diff --git a/analytics/src/db.js b/analytics/src/db.js index 21406e1..a43ace4 100644 --- a/analytics/src/db.js +++ b/analytics/src/db.js @@ -44,6 +44,12 @@ CREATE TABLE IF NOT EXISTS daily ( count INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (site, day, dim, key) ); +-- Merker für die Nachtverdichtung: Tage, die schon in daily stehen (macht runNightly idempotent). +CREATE TABLE IF NOT EXISTS agg_state ( + site TEXT NOT NULL, + day TEXT NOT NULL, + PRIMARY KEY (site, day) +); `; export function openDb(path) { diff --git a/analytics/src/heat.js b/analytics/src/heat.js new file mode 100644 index 0000000..8e35bc1 --- /dev/null +++ b/analytics/src/heat.js @@ -0,0 +1,71 @@ +// GET /a/heat?token= (öffentlicher Listener): Klickpunkte für das Heatmap-Overlay. +// Token (R3): ., e = Ablauf in Unix-Sekunden. +import crypto from 'node:crypto'; +import { hostOf } from './ingest.js'; + +const SITE_KEY = /^[a-z0-9]{16,40}$/; +const MAX_AHEAD = 30 * 60; // Sekunden +const POINTS = 5000; +const CLICK_DAYS = 30; + +export function signHeatToken({ siteKey, path, vp, exp }, secret) { + const part = Buffer.from(JSON.stringify({ s: siteKey, p: path, v: vp, e: exp })).toString('base64url'); + return `${part}.${crypto.createHmac('sha256', secret).update(part).digest('hex')}`; +} + +// Liefert { siteKey, path, vp, exp } oder null (Signatur, Format, abgelaufen, > 30 Min voraus). +export function verifyHeatToken(token, secret, nowMs) { + if (!secret || typeof token !== 'string') return null; + const m = /^([A-Za-z0-9_-]+)\.([0-9a-f]{64})$/.exec(token); + if (!m) return null; + const expect = crypto.createHmac('sha256', secret).update(m[1]).digest(); + const got = Buffer.from(m[2], 'hex'); + if (got.length !== expect.length || !crypto.timingSafeEqual(got, expect)) return null; + let o; + try { o = JSON.parse(Buffer.from(m[1], 'base64url').toString('utf8')); } catch { return null; } + if (!o || typeof o !== 'object' || typeof o.s !== 'string' || !SITE_KEY.test(o.s) + || typeof o.p !== 'string' || !o.p.startsWith('/') || !['m', 't', 'd'].includes(o.v) + || !Number.isInteger(o.e)) return null; + const nowS = nowMs / 1000; + if (o.e <= nowS || o.e > nowS + MAX_AHEAD) return null; + return { siteKey: o.s, path: o.p, vp: o.v, exp: o.e }; +} + +export function createHeat({ db, config = {}, now }) { + const clock = now || Date.now; + const getSite = db.prepare('SELECT domains FROM sites WHERE siteKey = ?'); + const getClicks = db.prepare('SELECT x, y FROM clicks WHERE site = ? AND path = ? AND vp = ? AND ts >= ? ORDER BY ts DESC LIMIT ?'); + + // CORS: nur die eigenen https-Domains der Website (das Overlay läuft auf der Kundenseite). + function cors(res, req, domains) { + const origin = req.headers.origin; + const host = hostOf(origin); + res.setHeader('Vary', 'Origin'); + if (host && domains.includes(host)) res.setHeader('Access-Control-Allow-Origin', origin); + } + const domainsOf = (site) => { try { return JSON.parse(site.domains); } catch { return []; } }; + + // true, wenn die Anfrage /a/heat betraf. + return function handle(req, res, url) { + if (url.pathname !== '/a/heat') return false; + const end = (status, body, extra = {}) => { + const b = body ? JSON.stringify(body) : ''; + res.writeHead(status, { 'Cache-Control': 'no-store', ...(body ? { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(b) } : {}), ...extra }); + res.end(b); + return true; + }; + if (req.method === 'OPTIONS') { + // Preflight nur bei eigener Domain mit CORS-Köpfen beantworten; Token steht in der URL, Daten gibt es erst beim GET. + res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS'); + return end(204); + } + if (req.method !== 'GET') return end(405, null, { Allow: 'GET, OPTIONS' }); + const t = verifyHeatToken(url.searchParams.get('token'), config.heatSecret, clock()); + const site = t && getSite.get(t.siteKey); + if (!t || !site) return end(403, { error: 'Ungültiger oder abgelaufener Link' }); + cors(res, req, domainsOf(site)); + const since = clock() - CLICK_DAYS * 86400 * 1000; + const points = getClicks.all(t.siteKey, t.path, t.vp, since, POINTS).map((r) => ({ x: r.x, y: r.y })); + return end(200, { points }); + }; +} diff --git a/analytics/src/index.js b/analytics/src/index.js index 5dbe737..5a24b04 100644 --- a/analytics/src/index.js +++ b/analytics/src/index.js @@ -4,6 +4,7 @@ import { loadConfig } from './config.js'; import { openDb } from './db.js'; import { createSalt } from './salt.js'; import { loadGeo } from './geo.js'; +import { scheduleNightly } from './jobs.js'; import { createPublicServer, createAdminServer } from './server.js'; const config = loadConfig(process.env); @@ -22,7 +23,11 @@ for (const [name, server, port] of servers) { server.listen(port, () => console.log(`Analytics-Dienst ${name} auf Port ${port}`)); } +// Nachtjob: täglich 02:30 Europe/Berlin und einmal 60 s nach dem Start +const nightly = scheduleNightly(db); + process.on('SIGTERM', () => { + nightly.stop(); let open = servers.length; const done = () => { if (--open === 0) { db.close(); process.exit(0); } }; for (const [, server] of servers) { server.close(done); server.closeIdleConnections?.(); } diff --git a/analytics/src/ingest.js b/analytics/src/ingest.js index f51c265..0ee2827 100644 --- a/analytics/src/ingest.js +++ b/analytics/src/ingest.js @@ -42,7 +42,7 @@ function clientIp(req) { } // Nur https ohne expliziten Port zählt; alles andere ergibt '' (kein Treffer). -function hostOf(value) { +export function hostOf(value) { if (typeof value !== 'string' || !value) return ''; try { const u = new URL(value); diff --git a/analytics/src/jobs.js b/analytics/src/jobs.js new file mode 100644 index 0000000..1219052 --- /dev/null +++ b/analytics/src/jobs.js @@ -0,0 +1,119 @@ +// Nachtjob: Rohdaten tageweise in `daily` verdichten, Fristen durchsetzen. +// Idempotenz: agg_state(site, day) merkt sich verdichtete Tage; ein Tag wird nie zweimal gezählt. +// Rohzeilen bleiben bis zur 30-Tage-Löschung bestehen (Statistik für heute/gestern liest sie live). +const DAY_FMT = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Berlin', year: 'numeric', month: '2-digit', day: '2-digit' }); +const WALL_FMT = new Intl.DateTimeFormat('en-GB', { + timeZone: 'Europe/Berlin', hourCycle: 'h23', year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit', +}); +const DAY_MS = 86400 * 1000; +export const RAW_DAYS = 30; +export const OFF_DAYS = 60; + +// Kalendertag (YYYY-MM-DD) in Europe/Berlin. +export const dayOf = (ts) => DAY_FMT.format(ts); + +// Verdichtet Rohzeilen EINES Tages zu [dim, key, count]-Tripeln. +export function aggregateRows(rows) { + const m = new Map(); + const add = (dim, key, n = 1) => { const k = `${dim}\u0000${key}`; m.set(k, (m.get(k) || 0) + n); }; + const visitors = new Set(); + for (const r of rows) { + if (r.type === 'pv') { + add('pv_total', ''); add('pv_path', r.path || ''); add('pv_ref', r.ref || ''); + add('pv_country', r.country || ''); add('pv_device', r.device || ''); + } else if (r.type === 'en') { + add('en_count', ''); add('en_sec_sum', '', r.sec || 0); add('scroll_sum', '', r.scroll || 0); + if (r.visitor) visitors.add(r.visitor); + } else if (r.type === 'ct') { + add('ct_kind', r.kind || ''); + if (r.visitor) visitors.add(r.visitor); + } else if (r.type === 'bn' && ['shown', 'accept', 'reject'].includes(r.kind)) { + add(`bn_${r.kind}`, ''); + } + } + if (visitors.size) add('visitors', '', visitors.size); + return [...m].filter(([, n]) => n > 0).map(([k, n]) => { const [dim, key] = k.split('\u0000'); return [dim, key, n]; }); +} + +export function runNightly(db, { now = Date.now() } = {}) { + const today = dayOf(now); + const rawCut = now - RAW_DAYS * DAY_MS; + const offCut = new Date(now - OFF_DAYS * DAY_MS).toISOString(); + const siteKeys = db.prepare(`SELECT DISTINCT site FROM events_raw UNION SELECT siteKey FROM sites`).all().map((r) => r.site); + const selRaw = db.prepare('SELECT ts, type, path, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts < ? ORDER BY ts'); + const selDone = db.prepare('SELECT day FROM agg_state WHERE site = ?'); + const insDaily = db.prepare(`INSERT INTO daily (site, day, dim, key, count) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(site, day, dim, key) DO UPDATE SET count = count + excluded.count`); + const insDone = db.prepare('INSERT OR IGNORE INTO agg_state (site, day) VALUES (?, ?)'); + const stats = { aggregated: 0, sitesDeleted: 0 }; + + const tx = (fn) => { + db.exec('BEGIN'); + try { fn(); db.exec('COMMIT'); } catch (e) { db.exec('ROLLBACK'); throw e; } + }; + + for (const site of siteKeys) { + tx(() => { + const done = new Set(selDone.all(site).map((r) => r.day)); + const byDay = new Map(); + for (const r of selRaw.all(site, now)) { + const d = dayOf(r.ts); + if (d >= today || done.has(d)) continue; + if (!byDay.has(d)) byDay.set(d, []); + byDay.get(d).push(r); + } + for (const [day, rows] of byDay) { + for (const [dim, key, n] of aggregateRows(rows)) insDaily.run(site, day, dim, key, n); + insDone.run(site, day); + stats.aggregated++; + } + // Fristen erst NACH der Verdichtung (Rohdaten und Klicks > 30 Tage) + db.prepare('DELETE FROM events_raw WHERE site = ? AND ts < ?').run(site, rawCut); + db.prepare('DELETE FROM clicks WHERE site = ? AND ts < ?').run(site, rawCut); + }); + } + // Klicks ganz ohne Website-Zeile / Ereignisse + db.prepare('DELETE FROM clicks WHERE ts < ?').run(rawCut); + + for (const { siteKey } of db.prepare('SELECT siteKey FROM sites WHERE offSince IS NOT NULL AND offSince < ?').all(offCut)) { + tx(() => { + for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) db.prepare(`DELETE FROM ${t} WHERE site = ?`).run(siteKey); + db.prepare('DELETE FROM sites WHERE siteKey = ?').run(siteKey); + stats.sitesDeleted++; + }); + } + return stats; +} + +// Offset (ms) von Europe/Berlin gegenüber UTC zum Zeitpunkt ts. +function berlinOffset(ts) { + const p = Object.fromEntries(WALL_FMT.formatToParts(ts).map((x) => [x.type, x.value])); + return Date.UTC(p.year, p.month - 1, p.day, p.hour, p.minute, p.second) - Math.floor(ts / 1000) * 1000; +} + +// Nächster Lauf um hour:minute Europe/Berlin (strikt nach now), als UTC-ms. +export function nextRunAt(now, hour = 2, minute = 30) { + const [y, m, d] = dayOf(now).split('-').map(Number); + const at = (dd) => { + const guess = Date.UTC(y, m - 1, dd, hour, minute); + return guess - berlinOffset(guess - berlinOffset(guess)); + }; + const t = at(d); + return t > now ? t : at(d + 1); +} + +// Plant runNightly täglich 02:30 Berlin und einmal kurz nach dem Start; stop() beendet alle Timer. +export function scheduleNightly(db, { startDelayMs = 60000, log = console } = {}) { + let timer = null; + let stopped = false; + const run = () => { + try { + const r = runNightly(db, { now: Date.now() }); + log.log(`Analytics-Nachtjob: ${r.aggregated} Tage verdichtet, ${r.sitesDeleted} Websites gelöscht`); + } catch (e) { log.error('Analytics-Nachtjob:', e.message); } + }; + const arm = (delay, fn) => { if (!stopped) timer = setTimeout(fn, delay); }; + const daily = () => { run(); arm(nextRunAt(Date.now()) - Date.now(), daily); }; + arm(startDelayMs, () => { run(); arm(nextRunAt(Date.now()) - Date.now(), daily); }); + return { stop: () => { stopped = true; clearTimeout(timer); } }; +} diff --git a/analytics/src/server.js b/analytics/src/server.js index 8b484c9..d9cc76e 100644 --- a/analytics/src/server.js +++ b/analytics/src/server.js @@ -2,6 +2,8 @@ import http from 'node:http'; import crypto from 'node:crypto'; import { createIngest } from './ingest.js'; import { createSalt } from './salt.js'; +import { createHeat } from './heat.js'; +import { computeStats } from './stats.js'; const MAX_ADMIN_BODY = 64 * 1024; const SITE_KEY = /^[a-z0-9]{16,40}$/; @@ -57,6 +59,8 @@ function validateSite(b) { function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged }) { const ingest = role === 'public' ? createIngest({ db, geo, salt: salt || createSalt({ now }), now }) : null; + const heat = role === 'public' ? createHeat({ db, config, now }) : null; + const clock = now || Date.now; const server = http.createServer(async (req, res) => { try { const url = new URL(req.url, 'http://x'); @@ -64,6 +68,7 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged const method = req.method; if (ingest && await ingest(req, res, url.pathname)) return; + if (heat && heat(req, res, url)) return; if (method === 'GET' && url.pathname === '/a/health') return send(res, 200, { ok: true }); @@ -80,6 +85,7 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged db.prepare('DELETE FROM events_raw WHERE site = ?').run(key); db.prepare('DELETE FROM clicks WHERE site = ?').run(key); db.prepare('DELETE FROM daily WHERE site = ?').run(key); + db.prepare('DELETE FROM agg_state WHERE site = ?').run(key); db.exec('COMMIT'); } catch (e) { db.exec('ROLLBACK'); @@ -100,6 +106,16 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged onSitesChanged?.(); return send(res, 200, { ok: true }); } + + if (role === 'admin' && method === 'GET' && parts[0] === 'a' && parts[1] === 'admin' && parts[2] === 'stats' && parts.length === 4) { + if (!tokenOk(req.headers.authorization, config.adminToken)) return send(res, 401, { error: 'Nicht autorisiert' }); + const key = parts[3]; + if (!SITE_KEY.test(key)) return send(res, 400, { error: 'Ungültiger Website-Schlüssel' }); + const r = url.searchParams.get('range') ?? '30'; + if (!['7', '30', '90'].includes(r)) return send(res, 400, { error: 'Ungültiger Zeitraum (7, 30 oder 90)' }); + if (!db.prepare('SELECT 1 FROM sites WHERE siteKey = ?').get(key)) return send(res, 404, { error: 'Website nicht gefunden' }); + return send(res, 200, computeStats(db, key, Number(r), clock())); + } return send(res, 404, { error: 'Nicht gefunden' }); } catch (e) { const status = e.status || 500; @@ -115,4 +131,4 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged export const createPublicServer = ({ config, db, geo, salt, now }) => createHandlerServer({ config, db, role: 'public', geo, salt, now }); // Interner Listener für die Website-Verwaltung. // onSitesChanged: wird nach jeder Änderung an Websites gerufen (public.invalidateSites()). -export const createAdminServer = ({ config, db, onSitesChanged }) => createHandlerServer({ config, db, role: 'admin', onSitesChanged }); +export const createAdminServer = ({ config, db, onSitesChanged, now }) => createHandlerServer({ config, db, role: 'admin', onSitesChanged, now }); diff --git a/analytics/src/stats.js b/analytics/src/stats.js new file mode 100644 index 0000000..b1cf5a2 --- /dev/null +++ b/analytics/src/stats.js @@ -0,0 +1,69 @@ +// Statistik für das Portal: Vergangenheit aus `daily`, noch nicht verdichtete Tage live aus den Rohdaten. +import { dayOf, aggregateRows } from './jobs.js'; + +const DAY_MS = 86400 * 1000; +const TOP = 20; + +// Liste der Kalendertage (YYYY-MM-DD), oldest first, endend am Berlin-Tag von `now`. +function dayList(now, range) { + const [y, m, d] = dayOf(now).split('-').map(Number); + const days = []; + for (let i = range - 1; i >= 0; i--) days.push(new Date(Date.UTC(y, m - 1, d - i)).toISOString().slice(0, 10)); + return days; +} + +const top = (map) => [...map].map(([key, count]) => ({ key, count })).filter((e) => e.count > 0) + .sort((a, b) => b.count - a.count || (a.key < b.key ? -1 : 1)).slice(0, TOP); + +export function computeStats(db, site, range, now = Date.now()) { + const days = dayList(now, range); + const first = days[0]; + const inRange = new Set(days); + const pvByDay = new Map(days.map((d) => [d, 0])); + const dims = {}; + const bump = (dim, key, n) => { + if (!dims[dim]) dims[dim] = new Map(); + dims[dim].set(key, (dims[dim].get(key) || 0) + n); + }; + const apply = (day, dim, key, n) => { + if (dim === 'pv_total') pvByDay.set(day, pvByDay.get(day) + n); + bump(dim, key, n); + }; + + const done = new Set(db.prepare('SELECT day FROM agg_state WHERE site = ? AND day >= ?').all(site, first).map((r) => r.day)); + for (const r of db.prepare('SELECT day, dim, key, count FROM daily WHERE site = ? AND day >= ?').all(site, first)) { + if (inRange.has(r.day)) apply(r.day, r.dim, r.key, r.count); + } + // Tage ohne agg_state (heute, ggf. gestern vor 02:30) live aus den Rohdaten + const live = new Map(); + const from = now - (range + 1) * DAY_MS; + for (const r of db.prepare('SELECT ts, type, path, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts >= ? AND ts <= ? ORDER BY ts') + .all(site, from, now)) { + const d = dayOf(r.ts); + if (!inRange.has(d) || done.has(d)) continue; + if (!live.has(d)) live.set(d, []); + live.get(d).push(r); + } + for (const [day, rows] of live) for (const [dim, key, n] of aggregateRows(rows)) apply(day, dim, key, n); + + const sum = (dim) => [...(dims[dim] || new Map()).values()].reduce((a, b) => a + b, 0); + const banner = { shown: sum('bn_shown'), accept: sum('bn_accept'), reject: sum('bn_reject') }; + const decided = banner.accept + banner.reject; + const contacts = top(dims.ct_kind || new Map()); + return { + totals: { + pv: sum('pv_total'), + // Summe der Tages-Besucher (kein tagesübergreifender Abgleich möglich: Salz wechselt täglich) + visitors: sum('visitors'), + contacts: sum('ct_kind'), + consentRate: decided ? banner.accept / decided : 0, + }, + series: days.map((day) => ({ day, pv: pvByDay.get(day) })), + pages: top(dims.pv_path || new Map()), + refs: top(dims.pv_ref || new Map()), + countries: top(dims.pv_country || new Map()), + devices: top(dims.pv_device || new Map()), + contacts, + banner, + }; +} diff --git a/analytics/test/heat.test.js b/analytics/test/heat.test.js new file mode 100644 index 0000000..d18f8cd --- /dev/null +++ b/analytics/test/heat.test.js @@ -0,0 +1,102 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { openDb } from '../src/db.js'; +import { createPublicServer, createAdminServer } from '../src/server.js'; +import { signHeatToken, verifyHeatToken } from '../src/heat.js'; + +const KEY = 'abcdef0123456789abcd'; +const OTHER = 'zzzzzzzz0123456789zz'; +const SECRET = 'geheim'; +const NOW = Date.UTC(2026, 9, 10, 10, 0, 0); +const exp = (s = 600) => Math.floor(NOW / 1000) + s; + +async function setup() { + const db = openDb(':memory:'); + const site = db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)'); + site.run(KEY, 'p', JSON.stringify(['kunde.de', 'www.kunde.de'])); + site.run(OTHER, 'q', JSON.stringify(['andere.de'])); + const click = db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,?,?)'); + click.run(KEY, NOW - 1000, '/', 'd', 0.1, 0.2); + click.run(KEY, NOW - 2000, '/', 'd', 0.3, 0.4); + click.run(KEY, NOW - 1000, '/', 'm', 0.9, 0.9); + click.run(KEY, NOW - 1000, '/x', 'd', 0.8, 0.8); + click.run(OTHER, NOW - 1000, '/', 'd', 0.7, 0.7); + click.run(KEY, NOW - 31 * 86400000, '/', 'd', 0.5, 0.5); + const server = createPublicServer({ config: { heatSecret: SECRET }, db, now: () => NOW }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const base = `http://127.0.0.1:${server.address().port}`; + const get = (token, headers = {}) => fetch(`${base}/a/heat?token=${encodeURIComponent(token)}`, { headers }); + return { db, get, base, close: () => server.close() }; +} +const tok = (o = {}, secret = SECRET) => signHeatToken({ siteKey: KEY, path: '/', vp: 'd', exp: exp(), ...o }, secret); + +test('gültiges Token: neueste Punkte der Seite/Viewport, 30 Tage, no-store, CORS nur eigene Domain', async () => { + const s = await setup(); + const r = await s.get(tok(), { Origin: 'https://www.kunde.de' }); + assert.equal(r.status, 200); + assert.equal(r.headers.get('cache-control'), 'no-store'); + assert.equal(r.headers.get('access-control-allow-origin'), 'https://www.kunde.de'); + assert.deepEqual(await r.json(), { points: [{ x: 0.1, y: 0.2 }, { x: 0.3, y: 0.4 }] }); + const fremd = await s.get(tok(), { Origin: 'https://evil.example' }); + assert.equal(fremd.status, 200); + assert.equal(fremd.headers.get('access-control-allow-origin'), null); + const http = await s.get(tok(), { Origin: 'http://kunde.de' }); + assert.equal(http.headers.get('access-control-allow-origin'), null); + assert.equal(r.headers.get('access-control-allow-origin') === '*', false); + s.close(); +}); + +test('maximal 5000 neueste Punkte', async () => { + const s = await setup(); + s.db.exec('BEGIN'); + const ins = s.db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,?,?)'); + for (let i = 0; i < 5100; i++) ins.run(KEY, NOW - 10000 + i, '/many', 'd', 0.5, i < 100 ? 0.01 : 0.99); + s.db.exec('COMMIT'); + const j = await (await s.get(tok({ path: '/many' }))).json(); + assert.equal(j.points.length, 5000); + assert.ok(j.points.every((p) => p.y === 0.99)); // die 100 ältesten fehlen + s.close(); +}); + +test('403: abgelaufen, > 30 Min voraus, falsches Secret, manipuliert, Müll, fremde/unbekannte Website', async () => { + const s = await setup(); + const bad = [ + tok({ exp: exp(-1) }), + tok({ exp: exp(30 * 60 + 5) }), + tok({}, 'anderes-secret'), + 'abc', '', 'a.b', 'x'.repeat(10) + '.' + 'f'.repeat(64), + tok({ siteKey: 'nichtvorhanden0123456' }), + ]; + // Manipulation: Payload tauschen, Signatur behalten + const [part, sig] = tok().split('.'); + for (const patch of [{ p: '/x' }, { v: 'm' }, { s: OTHER }]) { + const o = { ...JSON.parse(Buffer.from(part, 'base64url')), ...patch }; + bad.push(`${Buffer.from(JSON.stringify(o)).toString('base64url')}.${sig}`); + } + for (const t of bad) assert.equal((await s.get(t)).status, 403, t.slice(0, 30)); + assert.equal((await fetch(`${s.base}/a/heat`)).status, 403); + // Token der anderen Website liefert nur deren Punkte (nie die von KEY) + const j = await (await s.get(tok({ siteKey: OTHER }))).json(); + assert.deepEqual(j.points, [{ x: 0.7, y: 0.7 }]); + s.close(); +}); + +test('Token mit exp genau 30 Min voraus ist gültig; Helfer verify', () => { + assert.ok(verifyHeatToken(tok({ exp: exp(1800) }), SECRET, NOW)); + assert.equal(verifyHeatToken(tok({ exp: exp(1801) }), SECRET, NOW), null); + assert.equal(verifyHeatToken(tok(), '', NOW), null); // leeres Secret sperrt + assert.equal(verifyHeatToken(tok({ vp: 'x' }), SECRET, NOW), null); + assert.equal(verifyHeatToken(tok({ path: 'kein-slash' }), SECRET, NOW), null); + const t = tok(); + assert.match(t, /^[A-Za-z0-9_-]+\.[0-9a-f]{64}$/); + assert.deepEqual(JSON.parse(Buffer.from(t.split('.')[0], 'base64url')), { s: KEY, p: '/', v: 'd', e: exp() }); +}); + +test('/a/heat nur auf dem öffentlichen Listener', async () => { + const db = openDb(':memory:'); + const server = createAdminServer({ config: { heatSecret: SECRET, adminToken: 't' }, db }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const r = await fetch(`http://127.0.0.1:${server.address().port}/a/heat?token=${tok()}`); + assert.equal(r.status, 404); + server.close(); +}); diff --git a/analytics/test/stats.test.js b/analytics/test/stats.test.js new file mode 100644 index 0000000..ab33986 --- /dev/null +++ b/analytics/test/stats.test.js @@ -0,0 +1,179 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { openDb } from '../src/db.js'; +import { runNightly, nextRunAt, dayOf } from '../src/jobs.js'; +import { createAdminServer } from '../src/server.js'; + +const KEY = 'abcdef0123456789abcd'; +const OTHER = 'zzzzzzzz0123456789zz'; +const NOW = Date.UTC(2026, 9, 10, 10, 0, 0); // 10.10.2026 12:00 Berlin +const H = 3600 * 1000; +const D = 24 * H; + +function mk() { + const db = openDb(':memory:'); + db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(KEY, 'p', '["a.de"]'); + const ev = (o) => db.prepare(`INSERT INTO events_raw (site, ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind, host) + VALUES (@site,@ts,@type,@path,@vp,@ref,@country,@device,@visitor,@sec,@scroll,@kind,@host)`) + .run({ site: KEY, path: null, vp: null, ref: null, country: null, device: null, visitor: null, sec: null, scroll: null, kind: null, host: null, ...o }); + return { db, ev }; +} +const daily = (db, site = KEY) => db.prepare('SELECT day, dim, key, count FROM daily WHERE site = ? ORDER BY day, dim, key').all(site); +const get = (db, day, dim, key = '') => db.prepare('SELECT count FROM daily WHERE site=? AND day=? AND dim=? AND key=?').get(KEY, day, dim, key)?.count; + +test('Verdichtung: Dimensionen, nur Tage < heute, idempotent', () => { + const { db, ev } = mk(); + const t = NOW - D; // 9.10. + ev({ ts: t, type: 'pv', path: '/', vp: 'd', ref: 'google.com', country: 'DE', device: 'd' }); + ev({ ts: t + 1, type: 'pv', path: '/', vp: 'm', ref: null, country: 'DE', device: 'm' }); + ev({ ts: t + 2, type: 'pv', path: '/x', vp: 'd', ref: 'google.com', country: 'AT', device: 'd' }); + ev({ ts: t + 3, type: 'en', path: '/', vp: 'd', visitor: 'v1', sec: 10, scroll: 50 }); + ev({ ts: t + 4, type: 'en', path: '/x', vp: 'd', visitor: 'v1', sec: 20, scroll: 70 }); + ev({ ts: t + 5, type: 'ct', path: '/', visitor: 'v2', kind: 'tel' }); + ev({ ts: t + 6, type: 'ct', path: '/', visitor: 'v2', kind: 'out', host: 'x.de' }); + ev({ ts: t + 7, type: 'bn', kind: 'shown' }); + ev({ ts: t + 8, type: 'bn', kind: 'accept' }); + ev({ ts: NOW, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); // heute: unberührt + runNightly(db, { now: NOW }); + const d = '2026-10-09'; + assert.equal(get(db, d, 'pv_total'), 3); + assert.equal(get(db, d, 'pv_path', '/'), 2); + assert.equal(get(db, d, 'pv_path', '/x'), 1); + assert.equal(get(db, d, 'pv_ref', 'google.com'), 2); + assert.equal(get(db, d, 'pv_ref', ''), 1); + assert.equal(get(db, d, 'pv_country', 'DE'), 2); + assert.equal(get(db, d, 'pv_device', 'm'), 1); + assert.equal(get(db, d, 'visitors'), 2); // v1, v2 + assert.equal(get(db, d, 'en_count'), 2); + assert.equal(get(db, d, 'en_sec_sum'), 30); + assert.equal(get(db, d, 'scroll_sum'), 120); + assert.equal(get(db, d, 'ct_kind', 'tel'), 1); + assert.equal(get(db, d, 'ct_kind', 'out'), 1); + assert.equal(get(db, d, 'bn_shown'), 1); + assert.equal(get(db, d, 'bn_accept'), 1); + assert.equal(get(db, d, 'bn_reject'), undefined); + assert.equal(db.prepare("SELECT COUNT(*) c FROM daily WHERE day = '2026-10-10'").get().c, 0); + const before = daily(db); + runNightly(db, { now: NOW }); + runNightly(db, { now: NOW + H }); + assert.deepEqual(daily(db), before); +}); + +test('DST: 23:30 UTC im Sommer gehört zum nächsten Berlin-Tag', () => { + const { db, ev } = mk(); + ev({ ts: Date.UTC(2026, 6, 14, 23, 30), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); // 15.7. 01:30 Berlin + ev({ ts: Date.UTC(2026, 6, 14, 21, 59), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); // 14.7. 23:59 Berlin + assert.equal(dayOf(Date.UTC(2026, 6, 14, 23, 30)), '2026-07-15'); + runNightly(db, { now: Date.UTC(2026, 6, 16, 10) }); + assert.equal(get(db, '2026-07-14', 'pv_total'), 1); + assert.equal(get(db, '2026-07-15', 'pv_total'), 1); + // Winter: 23:30 UTC = 00:30 Berlin am Folgetag + assert.equal(dayOf(Date.UTC(2026, 0, 14, 23, 30)), '2026-01-15'); +}); + +test('Aufbewahrung: Rohdaten und Klicks > 30 Tage weg, vorher verdichtet', () => { + const { db, ev } = mk(); + ev({ ts: NOW - 31 * D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); + ev({ ts: NOW - 29 * D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); + const ins = db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,0.5,0.5)'); + ins.run(KEY, NOW - 31 * D, '/', 'd'); ins.run(KEY, NOW - 29 * D, '/', 'd'); + runNightly(db, { now: NOW }); + assert.equal(db.prepare('SELECT COUNT(*) c FROM events_raw').get().c, 1); + assert.equal(db.prepare('SELECT COUNT(*) c FROM clicks').get().c, 1); + assert.equal(get(db, dayOf(NOW - 31 * D), 'pv_total'), 1); // in daily erhalten +}); + +test('Website mit offSince > 60 Tage wird komplett gelöscht', () => { + const { db, ev } = mk(); + db.prepare('INSERT INTO sites (siteKey, projectId, domains, active, offSince) VALUES (?,?,?,0,?)') + .run(OTHER, 'q', '["b.de"]', new Date(NOW - 61 * D).toISOString()); + db.prepare('INSERT INTO sites (siteKey, projectId, domains, active, offSince) VALUES (?,?,?,0,?)') + .run('young0123456789abcdef', 'r', '["c.de"]', new Date(NOW - 59 * D).toISOString()); + ev({ site: OTHER, ts: NOW - D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); + ev({ ts: NOW - D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); + db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,0.1,0.1)').run(OTHER, NOW - D, '/', 'd'); + runNightly(db, { now: NOW }); + for (const t of ['sites', 'events_raw', 'clicks', 'daily', 'agg_state']) { + const col = t === 'sites' ? 'siteKey' : 'site'; + assert.equal(db.prepare(`SELECT COUNT(*) c FROM ${t} WHERE ${col} = ?`).get(OTHER).c, 0, t); + } + assert.equal(db.prepare('SELECT COUNT(*) c FROM sites').get().c, 2); // KEY + junge + assert.ok(get(db, dayOf(NOW - D), 'pv_total')); +}); + +test('nextRunAt: 02:30 Berlin (Sommer/Winter)', () => { + assert.equal(new Date(nextRunAt(Date.UTC(2026, 9, 1, 10))).toISOString(), '2026-10-02T00:30:00.000Z'); + assert.equal(new Date(nextRunAt(Date.UTC(2026, 9, 1, 0, 0))).toISOString(), '2026-10-01T00:30:00.000Z'); + assert.equal(new Date(nextRunAt(Date.UTC(2026, 0, 1, 12))).toISOString(), '2026-01-02T01:30:00.000Z'); +}); + +async function admin(db, now = () => NOW) { + const server = createAdminServer({ config: { adminToken: 'tok' }, db, now }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const base = `http://127.0.0.1:${server.address().port}`; + const get = (p, auth = 'Bearer tok') => fetch(base + p, { headers: auth ? { Authorization: auth } : {} }); + return { get, close: () => server.close() }; +} + +test('Stats-Endpunkt: Form, Serie lückenlos, heute live + Vergangenheit aus daily, consentRate', async () => { + const { db, ev } = mk(); + const y = NOW - D; + for (let i = 0; i < 3; i++) ev({ ts: y + i, type: 'pv', path: '/a', vp: 'd', ref: 'g.de', country: 'DE', device: 'd' }); + ev({ ts: y + 10, type: 'en', path: '/a', vp: 'd', visitor: 'v1', sec: 5, scroll: 5 }); + ev({ ts: y + 11, type: 'bn', kind: 'accept' }); + runNightly(db, { now: NOW }); // gestern -> daily + ev({ ts: NOW - H, type: 'pv', path: '/b', vp: 'm', country: 'AT', device: 'm' }); // heute live + ev({ ts: NOW - H, type: 'ct', path: '/b', visitor: 'v9', kind: 'mail' }); + ev({ ts: NOW - H, type: 'bn', kind: 'accept' }); + ev({ ts: NOW - H, type: 'bn', kind: 'reject' }); + ev({ ts: NOW - H, type: 'bn', kind: 'reject' }); + ev({ ts: NOW - H, type: 'bn', kind: 'shown' }); + const a = await admin(db); + const r = await a.get(`/a/admin/stats/${KEY}?range=7`); + assert.equal(r.status, 200); + const j = await r.json(); + assert.deepEqual(Object.keys(j).sort(), ['banner', 'contacts', 'countries', 'devices', 'pages', 'refs', 'series', 'totals']); + assert.equal(j.series.length, 7); + assert.equal(j.series[0].day, '2026-10-04'); + assert.equal(j.series[6].day, '2026-10-10'); + assert.deepEqual(j.series.map((s) => s.pv), [0, 0, 0, 0, 0, 3, 1]); + assert.deepEqual(j.totals, { pv: 4, visitors: 2, contacts: 1, consentRate: 2 / 4 }); + assert.deepEqual(j.pages, [{ key: '/a', count: 3 }, { key: '/b', count: 1 }]); + assert.deepEqual(j.refs[0], { key: 'g.de', count: 3 }); + assert.deepEqual(j.countries, [{ key: 'DE', count: 3 }, { key: 'AT', count: 1 }]); + assert.deepEqual(j.devices, [{ key: 'd', count: 3 }, { key: 'm', count: 1 }]); + assert.deepEqual(j.contacts, [{ key: 'mail', count: 1 }]); + assert.deepEqual(j.banner, { shown: 1, accept: 2, reject: 2 }); + // range 90: 90 Tage + const j90 = await (await a.get(`/a/admin/stats/${KEY}?range=90`)).json(); + assert.equal(j90.series.length, 90); + assert.equal(j90.totals.pv, 4); + a.close(); +}); + +test('Stats: gestern noch nicht verdichtet wird live gezählt, nie doppelt', async () => { + const { db, ev } = mk(); + ev({ ts: NOW - D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); + const a = await admin(db); + assert.equal((await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json()).totals.pv, 1); + runNightly(db, { now: NOW }); + assert.equal((await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json()).totals.pv, 1); + a.close(); +}); + +test('Stats: Top 20 absteigend, consentRate 0 ohne Entscheidungen, Fehlerfälle', async () => { + const { db, ev } = mk(); + for (let p = 0; p < 25; p++) for (let i = 0; i <= p; i++) ev({ ts: NOW - 1000, type: 'pv', path: `/p${p}`, vp: 'd', country: 'DE', device: 'd' }); + const a = await admin(db); + const j = await (await a.get(`/a/admin/stats/${KEY}?range=30`)).json(); + assert.equal(j.pages.length, 20); + assert.equal(j.pages[0].key, '/p24'); + assert.ok(j.pages.every((e, i, arr) => !i || arr[i - 1].count >= e.count)); + assert.equal(j.totals.consentRate, 0); + assert.equal((await a.get(`/a/admin/stats/${KEY}?range=5`)).status, 400); + assert.equal((await a.get(`/a/admin/stats/${KEY}?range=abc`)).status, 400); + assert.equal((await a.get(`/a/admin/stats/${OTHER}?range=7`)).status, 404); + assert.equal((await a.get(`/a/admin/stats/${KEY}?range=7`, null)).status, 401); + assert.equal((await a.get(`/a/admin/stats/${KEY}?range=7`, 'Bearer nope')).status, 401); + a.close(); +});