From 96409c6306b61866208190b30831355e4ca64d23 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 15:59:36 +0000 Subject: [PATCH] Analyse: Messdienst: Ping ohne Einwilligung, Zeitstempel auf Stunde, Pfade ohne Kennungen Co-Authored-By: Claude Opus 5.5 --- analytics/src/heat.js | 2 +- analytics/src/ingest.js | 28 ++++++++++--- analytics/src/jobs.js | 7 +++- analytics/src/stats.js | 2 +- analytics/test/heat.test.js | 14 +++++++ analytics/test/ingest.test.js | 75 +++++++++++++++++++++++++++++++++++ analytics/test/stats.test.js | 31 +++++++++++++++ 7 files changed, 149 insertions(+), 10 deletions(-) diff --git a/analytics/src/heat.js b/analytics/src/heat.js index 3e6d4cf..648e6e8 100644 --- a/analytics/src/heat.js +++ b/analytics/src/heat.js @@ -35,7 +35,7 @@ export function verifyHeatToken(token, secret, nowMs, { ignoreExpiry = false } = export function createHeat({ db, config = {}, now }) { const clock = now || Date.now; const getSite = db.prepare('SELECT domains, active FROM sites WHERE siteKey = ?'); - const getClicks = db.prepare('SELECT x, y FROM clicks WHERE site = ? AND path = ? AND vp = ? AND ts >= ? ORDER BY ts DESC LIMIT ?'); + const getClicks = db.prepare('SELECT x, y FROM clicks WHERE site = ? AND path = ? AND vp = ? AND ts >= ? ORDER BY ts DESC, id DESC LIMIT ?'); // CORS: nur die eigenen https-Domains der Website (das Overlay läuft auf der Kundenseite). function cors(res, req, domains) { diff --git a/analytics/src/ingest.js b/analytics/src/ingest.js index 0ee2827..fee894d 100644 --- a/analytics/src/ingest.js +++ b/analytics/src/ingest.js @@ -10,6 +10,7 @@ const HOUR = 3600 * 1000; const MINUTE = 60 * 1000; const FIELDS = Object.assign(Object.create(null), { pv: ['t', 'site', 'path', 'ref', 'vp'], + cx: ['t', 'site', 'path', 'ref', 'vp'], bn: ['t', 'site', 'a'], en: ['t', 'site', 'path', 'vp', 'sec', 'scroll'], ck: ['t', 'site', 'path', 'vp', 'x', 'y', 'sel'], @@ -55,11 +56,19 @@ function anyHostOf(value) { try { return new URL(value).hostname.toLowerCase(); } catch { return ''; } } +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +// Kennungsartige Segmente (lange Ziffernfolge, UUID, E-Mail) werden nie gespeichert. +function looksLikeId(seg) { + let d = seg; + try { d = decodeURIComponent(seg); } catch { /* Rohwert */ } + return /\d{6,}/.test(d) || UUID.test(d) || d.includes('@'); +} + function cleanPath(p) { if (typeof p !== 'string') return null; const s = p.split(/[?#]/)[0]; if (!s.startsWith('/') || s.length > 300 || /[\u0000-\u001f\u007f]/.test(s)) return null; - return s; + return s.split('/').map((seg) => (looksLikeId(seg) ? ':id' : seg)).join('/'); } function cleanHost(h) { @@ -134,12 +143,19 @@ export function createIngest({ db, geo = { lookup: () => '' }, salt, now = Date. } const path = cleanPath(e.path); if (!path) return; - if (e.t === 'pv') { + const hour = Math.floor(t / HOUR) * HOUR; + if (e.t === 'cx') { const vp = cleanVp(e.vp); if (!vp) return; + insEvent.run(site, hour, 'cx', path, vp, cleanRef(e.ref, domains), null, null, null, null, null, null, null); + return; + } + if (e.t === 'pv') { + const vp = e.vp === undefined ? null : cleanVp(e.vp); + if (e.vp !== undefined && !vp) return; const c = String(geo.lookup(ip) || ''); const country = /^[A-Za-z]{2}$/.test(c) ? c.toUpperCase() : ''; - insEvent.run(site, t, 'pv', path, vp, cleanRef(e.ref, domains), country, classifyUa(ua).device, null, null, null, null, null); + insEvent.run(site, hour, 'pv', path, vp, vp ? cleanRef(e.ref, domains) : null, country, classifyUa(ua).device, null, null, null, null, null); return; } if (e.t === 'ck') { @@ -148,7 +164,7 @@ export function createIngest({ db, geo = { lookup: () => '' }, salt, now = Date. const y = num(e.y); if (!vp || x === null || y === null) return; const sel = typeof e.sel === 'string' ? e.sel.slice(0, 200) : null; - insClick.run(site, t, path, vp, Math.round(clamp(x, 0, 1) * 1000) / 1000, Math.round(clamp(y, 0, 1) * 1000) / 1000, sel); + insClick.run(site, hour, path, vp, Math.round(clamp(x, 0, 1) * 1000) / 1000, Math.round(clamp(y, 0, 1) * 1000) / 1000, sel); return; } const visitor = salt.visitorId(site, ip, ua); @@ -157,14 +173,14 @@ export function createIngest({ db, geo = { lookup: () => '' }, salt, now = Date. const sec = num(e.sec); const scroll = num(e.scroll); if (!vp || sec === null || scroll === null) return; - insEvent.run(site, t, 'en', path, vp, null, null, null, visitor, Math.floor(clamp(sec, 0, 3600)), Math.floor(clamp(scroll, 0, 100)), null, null); + insEvent.run(site, hour, 'en', path, vp, null, null, null, visitor, Math.floor(clamp(sec, 0, 3600)), Math.floor(clamp(scroll, 0, 100)), null, null); return; } if (e.t === 'ct') { if (!['tel', 'mail', 'wa', 'out'].includes(e.kind)) return; let host = null; if (e.kind === 'out') { host = cleanHost(e.host); if (!host) return; } - insEvent.run(site, t, 'ct', path, null, null, null, null, visitor, null, null, e.kind, host); + insEvent.run(site, hour, 'ct', path, null, null, null, null, visitor, null, null, e.kind, host); } } diff --git a/analytics/src/jobs.js b/analytics/src/jobs.js index f82bacc..ccb8fbb 100644 --- a/analytics/src/jobs.js +++ b/analytics/src/jobs.js @@ -19,8 +19,11 @@ export function aggregateRows(rows) { const visitors = new Set(); for (const r of rows) { if (r.type === 'pv') { - add('pv_total', ''); add('pv_path', r.path || ''); add('pv_ref', r.ref || ''); + add('pv_total', ''); add('pv_path', r.path || ''); + if (r.vp != null) add('pv_ref', r.ref || ''); // nur Besucher mit Einwilligung; sonst kommt pv_ref aus cx add('pv_country', r.country || ''); add('pv_device', r.device || ''); + } else if (r.type === 'cx') { + add('pv_ref', r.ref || ''); } else if (r.type === 'en') { add('en_count', ''); add('en_sec_sum', '', r.sec || 0); add('scroll_sum', '', r.scroll || 0); if (r.visitor) visitors.add(r.visitor); @@ -40,7 +43,7 @@ export function runNightly(db, { now = Date.now() } = {}) { const rawCut = now - RAW_DAYS * DAY_MS; const offCut = new Date(now - OFF_DAYS * DAY_MS).toISOString(); const siteKeys = db.prepare(`SELECT DISTINCT site FROM events_raw UNION SELECT siteKey FROM sites`).all().map((r) => r.site); - const selRaw = db.prepare('SELECT ts, type, path, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts < ? ORDER BY ts'); + const selRaw = db.prepare('SELECT ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts < ? ORDER BY ts'); const selDone = db.prepare('SELECT day FROM agg_state WHERE site = ?'); const insDaily = db.prepare(`INSERT INTO daily (site, day, dim, key, count) VALUES (?, ?, ?, ?, ?) ON CONFLICT(site, day, dim, key) DO UPDATE SET count = count + excluded.count`); diff --git a/analytics/src/stats.js b/analytics/src/stats.js index b1cf5a2..e2033a9 100644 --- a/analytics/src/stats.js +++ b/analytics/src/stats.js @@ -37,7 +37,7 @@ export function computeStats(db, site, range, now = Date.now()) { // Tage ohne agg_state (heute, ggf. gestern vor 02:30) live aus den Rohdaten const live = new Map(); const from = now - (range + 1) * DAY_MS; - for (const r of db.prepare('SELECT ts, type, path, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts >= ? AND ts <= ? ORDER BY ts') + for (const r of db.prepare('SELECT ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts >= ? AND ts <= ? ORDER BY ts') .all(site, from, now)) { const d = dayOf(r.ts); if (!inRange.has(d) || done.has(d)) continue; diff --git a/analytics/test/heat.test.js b/analytics/test/heat.test.js index aec582b..de0d650 100644 --- a/analytics/test/heat.test.js +++ b/analytics/test/heat.test.js @@ -136,3 +136,17 @@ test('OPTIONS-Preflight: CORS-Köpfe nur für erlaubte Origins bekannter Website assert.equal((await pre('abc', 'https://kunde.de')).headers.get('access-control-allow-origin'), null); s.close(); }); + +test('Heatmap: gleiche Stunde (ts-Gleichstand), Limit schneidet über die niedrigste id ab', async () => { + const s = await setup(); + const ins = s.db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,?,?)'); + const ts = NOW - 3600000; + ins.run(KEY, ts, '/g', 'd', 0.99, 0.99); // niedrigste id + s.db.exec('BEGIN'); + for (let i = 0; i < 5000; i++) ins.run(KEY, ts, '/g', 'd', 0.5, 0.5); + s.db.exec('COMMIT'); + const j = await (await s.get(tok({ path: '/g' }))).json(); + assert.equal(j.points.length, 5000); + assert.ok(!j.points.some((p) => p.x === 0.99)); + s.close(); +}); diff --git a/analytics/test/ingest.test.js b/analytics/test/ingest.test.js index 0f2e27f..e79c595 100644 --- a/analytics/test/ingest.test.js +++ b/analytics/test/ingest.test.js @@ -350,3 +350,78 @@ test('X-Forwarded-For: rechtester Eintrag zählt für das Mengenlimit', async () assert.equal(t.rows().length, 121); t.close(); }); + +test('T2: pv ohne ref/vp wird gespeichert (vp NULL, ref NULL), Gerät und Land bleiben', async () => { + const t = await setup(); + await t.post({ t: 'pv', site: KEY, path: '/' }); + const r = t.rows(); + assert.equal(r.length, 1); + assert.equal(r[0].type, 'pv'); + assert.equal(r[0].vp, null); + assert.equal(r[0].ref, null); + assert.equal(r[0].device, 'm'); + assert.equal(r[0].country, 'DE'); + await t.post({ t: 'pv', site: KEY, path: '/', vp: 'x' }); + assert.equal(t.rows().length, 1); + t.close(); +}); + +test('T2: cx braucht gültiges vp, speichert ref, eigene Domain ergibt NULL', async () => { + const t = await setup(); + await t.post({ t: 'cx', site: KEY, path: '/', ref: 'https://www.google.com/x' }); + await t.post({ t: 'cx', site: KEY, path: '/', ref: 'google.com', vp: 'x' }); + assert.equal(t.rows().length, 0); + await t.post({ t: 'cx', site: KEY, path: '/', ref: 'https://www.google.com/x', vp: 'd' }); + await t.post({ t: 'cx', site: KEY, path: '/', ref: 'https://www.kunde.de/x', vp: 'm' }); + const r = t.rows(); + assert.equal(r.length, 2); + assert.equal(r[0].type, 'cx'); + assert.equal(r[0].ref, 'www.google.com'); + assert.equal(r[0].vp, 'd'); + assert.equal(r[0].country, null); + assert.equal(r[0].device, null); + assert.equal(r[1].ref, null); + t.close(); +}); + +test('T3: alle Zeitstempel auf die volle Stunde', async () => { + const t = await setup(); + const hour = Date.UTC(2026, 9, 1, 10, 0, 0); + await t.post(pv()); + await t.post({ t: 'cx', site: KEY, path: '/', vp: 'd' }); + await t.post({ t: 'en', site: KEY, path: '/', vp: 'd', sec: 5, scroll: 10 }); + await t.post({ t: 'ct', site: KEY, path: '/', kind: 'tel' }); + await t.post({ t: 'ck', site: KEY, path: '/', vp: 'd', x: 0.5, y: 0.5 }); + const r = t.rows(); + assert.deepEqual(r.map((x) => x.type), ['pv', 'cx', 'en', 'ct']); + for (const x of r) assert.equal(x.ts, hour); + const c = t.db.prepare('SELECT ts FROM clicks').all(); + assert.equal(c.length, 1); + assert.equal(c[0].ts, hour); + assert.equal(t.db.prepare('SELECT COUNT(*) n FROM events_raw WHERE ts % 3600000 != 0').get().n, 0); + t.close(); +}); + +test('T14: kennungsartige Pfadsegmente werden zu :id, alle Typen', async () => { + const t = await setup(); + const cases = [ + ['/kunde/123456789/rechnung', '/kunde/:id/rechnung'], + ['/u/3f2b8c1e-5d4a-4e0b-9a7c-1b2c3d4e5f60', '/u/:id'], + ['/mail/max%40example.com', '/mail/:id'], + ['/mail/max@example.com', '/mail/:id'], + ['/preise/2026', '/preise/2026'], + ['/blog/12345', '/blog/12345'], + ['/', '/'], + ['/a/123456?x=1#h', '/a/:id'], + ]; + for (const [p] of cases) await t.post(pv({ path: p })); + assert.deepEqual(t.rows().map((r) => r.path), cases.map((c) => c[1])); + await t.post({ t: 'cx', site: KEY, path: '/kunde/987654321', vp: 'd' }); + await t.post({ t: 'en', site: KEY, path: '/kunde/987654321', vp: 'd', sec: 1, scroll: 1 }); + await t.post({ t: 'ct', site: KEY, path: '/kunde/987654321', kind: 'tel' }); + await t.post({ t: 'ck', site: KEY, path: '/kunde/987654321', vp: 'd', x: 0.1, y: 0.1 }); + const paths = t.rows().slice(cases.length).map((r) => r.path); + assert.deepEqual(paths, ['/kunde/:id', '/kunde/:id', '/kunde/:id']); + assert.equal(t.db.prepare('SELECT path FROM clicks').get().path, '/kunde/:id'); + t.close(); +}); diff --git a/analytics/test/stats.test.js b/analytics/test/stats.test.js index 2e0f802..a9e07c2 100644 --- a/analytics/test/stats.test.js +++ b/analytics/test/stats.test.js @@ -204,3 +204,34 @@ test('scheduleNightly: früh feuernder Timer läuft nicht vorzeitig und nie dopp s.stop(); assert.equal(pending.length, 0); }); + +test('T2: pv ohne Einwilligung + cx: pv_total 1, pv_ref nur {google.com:1}; cx zählt nie pv_total', () => { + const { db, ev } = mk(); + const t = NOW - D; + ev({ ts: t, type: 'pv', path: '/', vp: null, ref: null, country: 'DE', device: 'd' }); + ev({ ts: t, type: 'cx', vp: 'd', ref: 'google.com' }); + ev({ ts: t, type: 'cx', vp: 'd', ref: null }); + ev({ ts: t, type: 'pv', path: '/', vp: 'd', ref: null, country: 'DE', device: 'd' }); + runNightly(db, { now: NOW }); + const d = '2026-10-09'; + assert.equal(get(db, d, 'pv_total'), 2); + assert.equal(get(db, d, 'pv_ref', 'google.com'), 1); + assert.equal(get(db, d, 'pv_ref', ''), 2); // pv mit vp und cx ohne ref + const refs = db.prepare("SELECT key FROM daily WHERE site=? AND dim='pv_ref' ORDER BY key").all(KEY).map((r) => r.key); + assert.deepEqual(refs, ['', 'google.com']); +}); + +test('T2: Stats-Endpunkt refs live und aus daily, nie doppelt', async () => { + const { db, ev } = mk(); + ev({ ts: NOW - D, type: 'pv', path: '/', vp: null, country: 'DE', device: 'd' }); + ev({ ts: NOW - D, type: 'cx', vp: 'd', ref: 'google.com' }); + const a = await admin(db); + const live = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json(); + assert.equal(live.totals.pv, 1); + assert.deepEqual(live.refs, [{ key: 'google.com', count: 1 }]); + runNightly(db, { now: NOW }); + const agg = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json(); + assert.equal(agg.totals.pv, 1); + assert.deepEqual(agg.refs, [{ key: 'google.com', count: 1 }]); + a.close(); +});