Analyse: Messdienst: Ping ohne Einwilligung, Zeitstempel auf Stunde, Pfade ohne Kennungen

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

analytics/src/heat.js

@@ -35,7 +35,7 @@ export function verifyHeatToken(token, secret, nowMs, { ignoreExpiry = false } =
 export function createHeat({ db, config = {}, now }) {
   const clock = now || Date.now;
   const getSite = db.prepare('SELECT domains, active FROM sites WHERE siteKey = ?');
-  const getClicks = db.prepare('SELECT x, y FROM clicks WHERE site = ? AND path = ? AND vp = ? AND ts >= ? ORDER BY ts DESC LIMIT ?');
+  const getClicks = db.prepare('SELECT x, y FROM clicks WHERE site = ? AND path = ? AND vp = ? AND ts >= ? ORDER BY ts DESC, id DESC LIMIT ?');
 
   // CORS: nur die eigenen https-Domains der Website (das Overlay läuft auf der Kundenseite).
   function cors(res, req, domains) {

analytics/src/ingest.js

@@ -10,6 +10,7 @@ const HOUR = 3600 * 1000;
 const MINUTE = 60 * 1000;
 const FIELDS = Object.assign(Object.create(null), {
   pv: ['t', 'site', 'path', 'ref', 'vp'],
+  cx: ['t', 'site', 'path', 'ref', 'vp'],
   bn: ['t', 'site', 'a'],
   en: ['t', 'site', 'path', 'vp', 'sec', 'scroll'],
   ck: ['t', 'site', 'path', 'vp', 'x', 'y', 'sel'],
@@ -55,11 +56,19 @@ function anyHostOf(value) {
   try { return new URL(value).hostname.toLowerCase(); } catch { return ''; }
 }
 
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+// Kennungsartige Segmente (lange Ziffernfolge, UUID, E-Mail) werden nie gespeichert.
+function looksLikeId(seg) {
+  let d = seg;
+  try { d = decodeURIComponent(seg); } catch { /* Rohwert */ }
+  return /\d{6,}/.test(d) || UUID.test(d) || d.includes('@');
+}
+
 function cleanPath(p) {
   if (typeof p !== 'string') return null;
   const s = p.split(/[?#]/)[0];
   if (!s.startsWith('/') || s.length > 300 || /[\u0000-\u001f\u007f]/.test(s)) return null;
-  return s;
+  return s.split('/').map((seg) => (looksLikeId(seg) ? ':id' : seg)).join('/');
 }
 
 function cleanHost(h) {
@@ -134,12 +143,19 @@ export function createIngest({ db, geo = { lookup: () => '' }, salt, now = Date.
     }
     const path = cleanPath(e.path);
     if (!path) return;
-    if (e.t === 'pv') {
+    const hour = Math.floor(t / HOUR) * HOUR;
+    if (e.t === 'cx') {
       const vp = cleanVp(e.vp);
       if (!vp) return;
+      insEvent.run(site, hour, 'cx', path, vp, cleanRef(e.ref, domains), null, null, null, null, null, null, null);
+      return;
+    }
+    if (e.t === 'pv') {
+      const vp = e.vp === undefined ? null : cleanVp(e.vp);
+      if (e.vp !== undefined && !vp) return;
       const c = String(geo.lookup(ip) || '');
       const country = /^[A-Za-z]{2}$/.test(c) ? c.toUpperCase() : '';
-      insEvent.run(site, t, 'pv', path, vp, cleanRef(e.ref, domains), country, classifyUa(ua).device, null, null, null, null, null);
+      insEvent.run(site, hour, 'pv', path, vp, vp ? cleanRef(e.ref, domains) : null, country, classifyUa(ua).device, null, null, null, null, null);
       return;
     }
     if (e.t === 'ck') {
@@ -148,7 +164,7 @@ export function createIngest({ db, geo = { lookup: () => '' }, salt, now = Date.
       const y = num(e.y);
       if (!vp || x === null || y === null) return;
       const sel = typeof e.sel === 'string' ? e.sel.slice(0, 200) : null;
-      insClick.run(site, t, path, vp, Math.round(clamp(x, 0, 1) * 1000) / 1000, Math.round(clamp(y, 0, 1) * 1000) / 1000, sel);
+      insClick.run(site, hour, path, vp, Math.round(clamp(x, 0, 1) * 1000) / 1000, Math.round(clamp(y, 0, 1) * 1000) / 1000, sel);
       return;
     }
     const visitor = salt.visitorId(site, ip, ua);
@@ -157,14 +173,14 @@ export function createIngest({ db, geo = { lookup: () => '' }, salt, now = Date.
       const sec = num(e.sec);
       const scroll = num(e.scroll);
       if (!vp || sec === null || scroll === null) return;
-      insEvent.run(site, t, 'en', path, vp, null, null, null, visitor, Math.floor(clamp(sec, 0, 3600)), Math.floor(clamp(scroll, 0, 100)), null, null);
+      insEvent.run(site, hour, 'en', path, vp, null, null, null, visitor, Math.floor(clamp(sec, 0, 3600)), Math.floor(clamp(scroll, 0, 100)), null, null);
       return;
     }
     if (e.t === 'ct') {
       if (!['tel', 'mail', 'wa', 'out'].includes(e.kind)) return;
       let host = null;
       if (e.kind === 'out') { host = cleanHost(e.host); if (!host) return; }
-      insEvent.run(site, t, 'ct', path, null, null, null, null, visitor, null, null, e.kind, host);
+      insEvent.run(site, hour, 'ct', path, null, null, null, null, visitor, null, null, e.kind, host);
     }
   }
 

analytics/src/jobs.js

@@ -19,8 +19,11 @@ export function aggregateRows(rows) {
   const visitors = new Set();
   for (const r of rows) {
     if (r.type === 'pv') {
-      add('pv_total', ''); add('pv_path', r.path || ''); add('pv_ref', r.ref || '');
+      add('pv_total', ''); add('pv_path', r.path || '');
+      if (r.vp != null) add('pv_ref', r.ref || ''); // nur Besucher mit Einwilligung; sonst kommt pv_ref aus cx
       add('pv_country', r.country || ''); add('pv_device', r.device || '');
+    } else if (r.type === 'cx') {
+      add('pv_ref', r.ref || '');
     } else if (r.type === 'en') {
       add('en_count', ''); add('en_sec_sum', '', r.sec || 0); add('scroll_sum', '', r.scroll || 0);
       if (r.visitor) visitors.add(r.visitor);
@@ -40,7 +43,7 @@ export function runNightly(db, { now = Date.now() } = {}) {
   const rawCut = now - RAW_DAYS * DAY_MS;
   const offCut = new Date(now - OFF_DAYS * DAY_MS).toISOString();
   const siteKeys = db.prepare(`SELECT DISTINCT site FROM events_raw UNION SELECT siteKey FROM sites`).all().map((r) => r.site);
-  const selRaw = db.prepare('SELECT ts, type, path, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts < ? ORDER BY ts');
+  const selRaw = db.prepare('SELECT ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts < ? ORDER BY ts');
   const selDone = db.prepare('SELECT day FROM agg_state WHERE site = ?');
   const insDaily = db.prepare(`INSERT INTO daily (site, day, dim, key, count) VALUES (?, ?, ?, ?, ?)
     ON CONFLICT(site, day, dim, key) DO UPDATE SET count = count + excluded.count`);

analytics/src/stats.js

@@ -37,7 +37,7 @@ export function computeStats(db, site, range, now = Date.now()) {
   // Tage ohne agg_state (heute, ggf. gestern vor 02:30) live aus den Rohdaten
   const live = new Map();
   const from = now - (range + 1) * DAY_MS;
-  for (const r of db.prepare('SELECT ts, type, path, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts >= ? AND ts <= ? ORDER BY ts')
+  for (const r of db.prepare('SELECT ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts >= ? AND ts <= ? ORDER BY ts')
     .all(site, from, now)) {
     const d = dayOf(r.ts);
     if (!inRange.has(d) || done.has(d)) continue;

analytics/test/heat.test.js

@@ -136,3 +136,17 @@ test('OPTIONS-Preflight: CORS-Köpfe nur für erlaubte Origins bekannter Website
   assert.equal((await pre('abc', 'https://kunde.de')).headers.get('access-control-allow-origin'), null);
   s.close();
 });
+
+test('Heatmap: gleiche Stunde (ts-Gleichstand), Limit schneidet über die niedrigste id ab', async () => {
+  const s = await setup();
+  const ins = s.db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,?,?)');
+  const ts = NOW - 3600000;
+  ins.run(KEY, ts, '/g', 'd', 0.99, 0.99); // niedrigste id
+  s.db.exec('BEGIN');
+  for (let i = 0; i < 5000; i++) ins.run(KEY, ts, '/g', 'd', 0.5, 0.5);
+  s.db.exec('COMMIT');
+  const j = await (await s.get(tok({ path: '/g' }))).json();
+  assert.equal(j.points.length, 5000);
+  assert.ok(!j.points.some((p) => p.x === 0.99));
+  s.close();
+});

analytics/test/ingest.test.js

@@ -350,3 +350,78 @@ test('X-Forwarded-For: rechtester Eintrag zählt für das Mengenlimit', async ()
   assert.equal(t.rows().length, 121);
   t.close();
 });
+
+test('T2: pv ohne ref/vp wird gespeichert (vp NULL, ref NULL), Gerät und Land bleiben', async () => {
+  const t = await setup();
+  await t.post({ t: 'pv', site: KEY, path: '/' });
+  const r = t.rows();
+  assert.equal(r.length, 1);
+  assert.equal(r[0].type, 'pv');
+  assert.equal(r[0].vp, null);
+  assert.equal(r[0].ref, null);
+  assert.equal(r[0].device, 'm');
+  assert.equal(r[0].country, 'DE');
+  await t.post({ t: 'pv', site: KEY, path: '/', vp: 'x' });
+  assert.equal(t.rows().length, 1);
+  t.close();
+});
+
+test('T2: cx braucht gültiges vp, speichert ref, eigene Domain ergibt NULL', async () => {
+  const t = await setup();
+  await t.post({ t: 'cx', site: KEY, path: '/', ref: 'https://www.google.com/x' });
+  await t.post({ t: 'cx', site: KEY, path: '/', ref: 'google.com', vp: 'x' });
+  assert.equal(t.rows().length, 0);
+  await t.post({ t: 'cx', site: KEY, path: '/', ref: 'https://www.google.com/x', vp: 'd' });
+  await t.post({ t: 'cx', site: KEY, path: '/', ref: 'https://www.kunde.de/x', vp: 'm' });
+  const r = t.rows();
+  assert.equal(r.length, 2);
+  assert.equal(r[0].type, 'cx');
+  assert.equal(r[0].ref, 'www.google.com');
+  assert.equal(r[0].vp, 'd');
+  assert.equal(r[0].country, null);
+  assert.equal(r[0].device, null);
+  assert.equal(r[1].ref, null);
+  t.close();
+});
+
+test('T3: alle Zeitstempel auf die volle Stunde', async () => {
+  const t = await setup();
+  const hour = Date.UTC(2026, 9, 1, 10, 0, 0);
+  await t.post(pv());
+  await t.post({ t: 'cx', site: KEY, path: '/', vp: 'd' });
+  await t.post({ t: 'en', site: KEY, path: '/', vp: 'd', sec: 5, scroll: 10 });
+  await t.post({ t: 'ct', site: KEY, path: '/', kind: 'tel' });
+  await t.post({ t: 'ck', site: KEY, path: '/', vp: 'd', x: 0.5, y: 0.5 });
+  const r = t.rows();
+  assert.deepEqual(r.map((x) => x.type), ['pv', 'cx', 'en', 'ct']);
+  for (const x of r) assert.equal(x.ts, hour);
+  const c = t.db.prepare('SELECT ts FROM clicks').all();
+  assert.equal(c.length, 1);
+  assert.equal(c[0].ts, hour);
+  assert.equal(t.db.prepare('SELECT COUNT(*) n FROM events_raw WHERE ts % 3600000 != 0').get().n, 0);
+  t.close();
+});
+
+test('T14: kennungsartige Pfadsegmente werden zu :id, alle Typen', async () => {
+  const t = await setup();
+  const cases = [
+    ['/kunde/123456789/rechnung', '/kunde/:id/rechnung'],
+    ['/u/3f2b8c1e-5d4a-4e0b-9a7c-1b2c3d4e5f60', '/u/:id'],
+    ['/mail/max%40example.com', '/mail/:id'],
+    ['/mail/max@example.com', '/mail/:id'],
+    ['/preise/2026', '/preise/2026'],
+    ['/blog/12345', '/blog/12345'],
+    ['/', '/'],
+    ['/a/123456?x=1#h', '/a/:id'],
+  ];
+  for (const [p] of cases) await t.post(pv({ path: p }));
+  assert.deepEqual(t.rows().map((r) => r.path), cases.map((c) => c[1]));
+  await t.post({ t: 'cx', site: KEY, path: '/kunde/987654321', vp: 'd' });
+  await t.post({ t: 'en', site: KEY, path: '/kunde/987654321', vp: 'd', sec: 1, scroll: 1 });
+  await t.post({ t: 'ct', site: KEY, path: '/kunde/987654321', kind: 'tel' });
+  await t.post({ t: 'ck', site: KEY, path: '/kunde/987654321', vp: 'd', x: 0.1, y: 0.1 });
+  const paths = t.rows().slice(cases.length).map((r) => r.path);
+  assert.deepEqual(paths, ['/kunde/:id', '/kunde/:id', '/kunde/:id']);
+  assert.equal(t.db.prepare('SELECT path FROM clicks').get().path, '/kunde/:id');
+  t.close();
+});

analytics/test/stats.test.js

@@ -204,3 +204,34 @@ test('scheduleNightly: früh feuernder Timer läuft nicht vorzeitig und nie dopp
   s.stop();
   assert.equal(pending.length, 0);
 });
+
+test('T2: pv ohne Einwilligung + cx: pv_total 1, pv_ref nur {google.com:1}; cx zählt nie pv_total', () => {
+  const { db, ev } = mk();
+  const t = NOW - D;
+  ev({ ts: t, type: 'pv', path: '/', vp: null, ref: null, country: 'DE', device: 'd' });
+  ev({ ts: t, type: 'cx', vp: 'd', ref: 'google.com' });
+  ev({ ts: t, type: 'cx', vp: 'd', ref: null });
+  ev({ ts: t, type: 'pv', path: '/', vp: 'd', ref: null, country: 'DE', device: 'd' });
+  runNightly(db, { now: NOW });
+  const d = '2026-10-09';
+  assert.equal(get(db, d, 'pv_total'), 2);
+  assert.equal(get(db, d, 'pv_ref', 'google.com'), 1);
+  assert.equal(get(db, d, 'pv_ref', ''), 2); // pv mit vp und cx ohne ref
+  const refs = db.prepare("SELECT key FROM daily WHERE site=? AND dim='pv_ref' ORDER BY key").all(KEY).map((r) => r.key);
+  assert.deepEqual(refs, ['', 'google.com']);
+});
+
+test('T2: Stats-Endpunkt refs live und aus daily, nie doppelt', async () => {
+  const { db, ev } = mk();
+  ev({ ts: NOW - D, type: 'pv', path: '/', vp: null, country: 'DE', device: 'd' });
+  ev({ ts: NOW - D, type: 'cx', vp: 'd', ref: 'google.com' });
+  const a = await admin(db);
+  const live = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json();
+  assert.equal(live.totals.pv, 1);
+  assert.deepEqual(live.refs, [{ key: 'google.com', count: 1 }]);
+  runNightly(db, { now: NOW });
+  const agg = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json();
+  assert.equal(agg.totals.pv, 1);
+  assert.deepEqual(agg.refs, [{ key: 'google.com', count: 1 }]);
+  a.close();
+});