Analytics: Verdichtung, Statistik, Heatmap-Daten

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

analytics/src/db.js

@@ -44,6 +44,12 @@ CREATE TABLE IF NOT EXISTS daily (
   count INTEGER NOT NULL DEFAULT 0,
   PRIMARY KEY (site, day, dim, key)
 );
+-- Merker für die Nachtverdichtung: Tage, die schon in daily stehen (macht runNightly idempotent).
+CREATE TABLE IF NOT EXISTS agg_state (
+  site TEXT NOT NULL,
+  day TEXT NOT NULL,
+  PRIMARY KEY (site, day)
+);
 `;
 
 export function openDb(path) {

analytics/src/heat.js

@@ -0,0 +1,71 @@
+// GET /a/heat?token= (öffentlicher Listener): Klickpunkte für das Heatmap-Overlay.
+// Token (R3): <b64url(JSON {s,p,v,e})>.<hex HMAC-SHA256(heatSecret, b64part)>, e = Ablauf in Unix-Sekunden.
+import crypto from 'node:crypto';
+import { hostOf } from './ingest.js';
+
+const SITE_KEY = /^[a-z0-9]{16,40}$/;
+const MAX_AHEAD = 30 * 60; // Sekunden
+const POINTS = 5000;
+const CLICK_DAYS = 30;
+
+export function signHeatToken({ siteKey, path, vp, exp }, secret) {
+  const part = Buffer.from(JSON.stringify({ s: siteKey, p: path, v: vp, e: exp })).toString('base64url');
+  return `${part}.${crypto.createHmac('sha256', secret).update(part).digest('hex')}`;
+}
+
+// Liefert { siteKey, path, vp, exp } oder null (Signatur, Format, abgelaufen, > 30 Min voraus).
+export function verifyHeatToken(token, secret, nowMs) {
+  if (!secret || typeof token !== 'string') return null;
+  const m = /^([A-Za-z0-9_-]+)\.([0-9a-f]{64})$/.exec(token);
+  if (!m) return null;
+  const expect = crypto.createHmac('sha256', secret).update(m[1]).digest();
+  const got = Buffer.from(m[2], 'hex');
+  if (got.length !== expect.length || !crypto.timingSafeEqual(got, expect)) return null;
+  let o;
+  try { o = JSON.parse(Buffer.from(m[1], 'base64url').toString('utf8')); } catch { return null; }
+  if (!o || typeof o !== 'object' || typeof o.s !== 'string' || !SITE_KEY.test(o.s)
+    || typeof o.p !== 'string' || !o.p.startsWith('/') || !['m', 't', 'd'].includes(o.v)
+    || !Number.isInteger(o.e)) return null;
+  const nowS = nowMs / 1000;
+  if (o.e <= nowS || o.e > nowS + MAX_AHEAD) return null;
+  return { siteKey: o.s, path: o.p, vp: o.v, exp: o.e };
+}
+
+export function createHeat({ db, config = {}, now }) {
+  const clock = now || Date.now;
+  const getSite = db.prepare('SELECT domains FROM sites WHERE siteKey = ?');
+  const getClicks = db.prepare('SELECT x, y FROM clicks WHERE site = ? AND path = ? AND vp = ? AND ts >= ? ORDER BY ts DESC LIMIT ?');
+
+  // CORS: nur die eigenen https-Domains der Website (das Overlay läuft auf der Kundenseite).
+  function cors(res, req, domains) {
+    const origin = req.headers.origin;
+    const host = hostOf(origin);
+    res.setHeader('Vary', 'Origin');
+    if (host && domains.includes(host)) res.setHeader('Access-Control-Allow-Origin', origin);
+  }
+  const domainsOf = (site) => { try { return JSON.parse(site.domains); } catch { return []; } };
+
+  // true, wenn die Anfrage /a/heat betraf.
+  return function handle(req, res, url) {
+    if (url.pathname !== '/a/heat') return false;
+    const end = (status, body, extra = {}) => {
+      const b = body ? JSON.stringify(body) : '';
+      res.writeHead(status, { 'Cache-Control': 'no-store', ...(body ? { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(b) } : {}), ...extra });
+      res.end(b);
+      return true;
+    };
+    if (req.method === 'OPTIONS') {
+      // Preflight nur bei eigener Domain mit CORS-Köpfen beantworten; Token steht in der URL, Daten gibt es erst beim GET.
+      res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
+      return end(204);
+    }
+    if (req.method !== 'GET') return end(405, null, { Allow: 'GET, OPTIONS' });
+    const t = verifyHeatToken(url.searchParams.get('token'), config.heatSecret, clock());
+    const site = t && getSite.get(t.siteKey);
+    if (!t || !site) return end(403, { error: 'Ungültiger oder abgelaufener Link' });
+    cors(res, req, domainsOf(site));
+    const since = clock() - CLICK_DAYS * 86400 * 1000;
+    const points = getClicks.all(t.siteKey, t.path, t.vp, since, POINTS).map((r) => ({ x: r.x, y: r.y }));
+    return end(200, { points });
+  };
+}

analytics/src/index.js

@@ -4,6 +4,7 @@ import { loadConfig } from './config.js';
 import { openDb } from './db.js';
 import { createSalt } from './salt.js';
 import { loadGeo } from './geo.js';
+import { scheduleNightly } from './jobs.js';
 import { createPublicServer, createAdminServer } from './server.js';
 
 const config = loadConfig(process.env);
@@ -22,7 +23,11 @@ for (const [name, server, port] of servers) {
   server.listen(port, () => console.log(`Analytics-Dienst ${name} auf Port ${port}`));
 }
 
+// Nachtjob: täglich 02:30 Europe/Berlin und einmal 60 s nach dem Start
+const nightly = scheduleNightly(db);
+
 process.on('SIGTERM', () => {
+  nightly.stop();
   let open = servers.length;
   const done = () => { if (--open === 0) { db.close(); process.exit(0); } };
   for (const [, server] of servers) { server.close(done); server.closeIdleConnections?.(); }

analytics/src/ingest.js

@@ -42,7 +42,7 @@ function clientIp(req) {
 }
 
 // Nur https ohne expliziten Port zählt; alles andere ergibt '' (kein Treffer).
-function hostOf(value) {
+export function hostOf(value) {
   if (typeof value !== 'string' || !value) return '';
   try {
     const u = new URL(value);

analytics/src/jobs.js

@@ -0,0 +1,119 @@
+// Nachtjob: Rohdaten tageweise in `daily` verdichten, Fristen durchsetzen.
+// Idempotenz: agg_state(site, day) merkt sich verdichtete Tage; ein Tag wird nie zweimal gezählt.
+// Rohzeilen bleiben bis zur 30-Tage-Löschung bestehen (Statistik für heute/gestern liest sie live).
+const DAY_FMT = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Berlin', year: 'numeric', month: '2-digit', day: '2-digit' });
+const WALL_FMT = new Intl.DateTimeFormat('en-GB', {
+  timeZone: 'Europe/Berlin', hourCycle: 'h23', year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit',
+});
+const DAY_MS = 86400 * 1000;
+export const RAW_DAYS = 30;
+export const OFF_DAYS = 60;
+
+// Kalendertag (YYYY-MM-DD) in Europe/Berlin.
+export const dayOf = (ts) => DAY_FMT.format(ts);
+
+// Verdichtet Rohzeilen EINES Tages zu [dim, key, count]-Tripeln.
+export function aggregateRows(rows) {
+  const m = new Map();
+  const add = (dim, key, n = 1) => { const k = `${dim}\u0000${key}`; m.set(k, (m.get(k) || 0) + n); };
+  const visitors = new Set();
+  for (const r of rows) {
+    if (r.type === 'pv') {
+      add('pv_total', ''); add('pv_path', r.path || ''); add('pv_ref', r.ref || '');
+      add('pv_country', r.country || ''); add('pv_device', r.device || '');
+    } else if (r.type === 'en') {
+      add('en_count', ''); add('en_sec_sum', '', r.sec || 0); add('scroll_sum', '', r.scroll || 0);
+      if (r.visitor) visitors.add(r.visitor);
+    } else if (r.type === 'ct') {
+      add('ct_kind', r.kind || '');
+      if (r.visitor) visitors.add(r.visitor);
+    } else if (r.type === 'bn' && ['shown', 'accept', 'reject'].includes(r.kind)) {
+      add(`bn_${r.kind}`, '');
+    }
+  }
+  if (visitors.size) add('visitors', '', visitors.size);
+  return [...m].filter(([, n]) => n > 0).map(([k, n]) => { const [dim, key] = k.split('\u0000'); return [dim, key, n]; });
+}
+
+export function runNightly(db, { now = Date.now() } = {}) {
+  const today = dayOf(now);
+  const rawCut = now - RAW_DAYS * DAY_MS;
+  const offCut = new Date(now - OFF_DAYS * DAY_MS).toISOString();
+  const siteKeys = db.prepare(`SELECT DISTINCT site FROM events_raw UNION SELECT siteKey FROM sites`).all().map((r) => r.site);
+  const selRaw = db.prepare('SELECT ts, type, path, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts < ? ORDER BY ts');
+  const selDone = db.prepare('SELECT day FROM agg_state WHERE site = ?');
+  const insDaily = db.prepare(`INSERT INTO daily (site, day, dim, key, count) VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(site, day, dim, key) DO UPDATE SET count = count + excluded.count`);
+  const insDone = db.prepare('INSERT OR IGNORE INTO agg_state (site, day) VALUES (?, ?)');
+  const stats = { aggregated: 0, sitesDeleted: 0 };
+
+  const tx = (fn) => {
+    db.exec('BEGIN');
+    try { fn(); db.exec('COMMIT'); } catch (e) { db.exec('ROLLBACK'); throw e; }
+  };
+
+  for (const site of siteKeys) {
+    tx(() => {
+      const done = new Set(selDone.all(site).map((r) => r.day));
+      const byDay = new Map();
+      for (const r of selRaw.all(site, now)) {
+        const d = dayOf(r.ts);
+        if (d >= today || done.has(d)) continue;
+        if (!byDay.has(d)) byDay.set(d, []);
+        byDay.get(d).push(r);
+      }
+      for (const [day, rows] of byDay) {
+        for (const [dim, key, n] of aggregateRows(rows)) insDaily.run(site, day, dim, key, n);
+        insDone.run(site, day);
+        stats.aggregated++;
+      }
+      // Fristen erst NACH der Verdichtung (Rohdaten und Klicks > 30 Tage)
+      db.prepare('DELETE FROM events_raw WHERE site = ? AND ts < ?').run(site, rawCut);
+      db.prepare('DELETE FROM clicks WHERE site = ? AND ts < ?').run(site, rawCut);
+    });
+  }
+  // Klicks ganz ohne Website-Zeile / Ereignisse
+  db.prepare('DELETE FROM clicks WHERE ts < ?').run(rawCut);
+
+  for (const { siteKey } of db.prepare('SELECT siteKey FROM sites WHERE offSince IS NOT NULL AND offSince < ?').all(offCut)) {
+    tx(() => {
+      for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) db.prepare(`DELETE FROM ${t} WHERE site = ?`).run(siteKey);
+      db.prepare('DELETE FROM sites WHERE siteKey = ?').run(siteKey);
+      stats.sitesDeleted++;
+    });
+  }
+  return stats;
+}
+
+// Offset (ms) von Europe/Berlin gegenüber UTC zum Zeitpunkt ts.
+function berlinOffset(ts) {
+  const p = Object.fromEntries(WALL_FMT.formatToParts(ts).map((x) => [x.type, x.value]));
+  return Date.UTC(p.year, p.month - 1, p.day, p.hour, p.minute, p.second) - Math.floor(ts / 1000) * 1000;
+}
+
+// Nächster Lauf um hour:minute Europe/Berlin (strikt nach now), als UTC-ms.
+export function nextRunAt(now, hour = 2, minute = 30) {
+  const [y, m, d] = dayOf(now).split('-').map(Number);
+  const at = (dd) => {
+    const guess = Date.UTC(y, m - 1, dd, hour, minute);
+    return guess - berlinOffset(guess - berlinOffset(guess));
+  };
+  const t = at(d);
+  return t > now ? t : at(d + 1);
+}
+
+// Plant runNightly täglich 02:30 Berlin und einmal kurz nach dem Start; stop() beendet alle Timer.
+export function scheduleNightly(db, { startDelayMs = 60000, log = console } = {}) {
+  let timer = null;
+  let stopped = false;
+  const run = () => {
+    try {
+      const r = runNightly(db, { now: Date.now() });
+      log.log(`Analytics-Nachtjob: ${r.aggregated} Tage verdichtet, ${r.sitesDeleted} Websites gelöscht`);
+    } catch (e) { log.error('Analytics-Nachtjob:', e.message); }
+  };
+  const arm = (delay, fn) => { if (!stopped) timer = setTimeout(fn, delay); };
+  const daily = () => { run(); arm(nextRunAt(Date.now()) - Date.now(), daily); };
+  arm(startDelayMs, () => { run(); arm(nextRunAt(Date.now()) - Date.now(), daily); });
+  return { stop: () => { stopped = true; clearTimeout(timer); } };
+}

analytics/src/server.js

@@ -2,6 +2,8 @@ import http from 'node:http';
 import crypto from 'node:crypto';
 import { createIngest } from './ingest.js';
 import { createSalt } from './salt.js';
+import { createHeat } from './heat.js';
+import { computeStats } from './stats.js';
 
 const MAX_ADMIN_BODY = 64 * 1024;
 const SITE_KEY = /^[a-z0-9]{16,40}$/;
@@ -57,6 +59,8 @@ function validateSite(b) {
 
 function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged }) {
   const ingest = role === 'public' ? createIngest({ db, geo, salt: salt || createSalt({ now }), now }) : null;
+  const heat = role === 'public' ? createHeat({ db, config, now }) : null;
+  const clock = now || Date.now;
   const server = http.createServer(async (req, res) => {
     try {
       const url = new URL(req.url, 'http://x');
@@ -64,6 +68,7 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged
       const method = req.method;
 
       if (ingest && await ingest(req, res, url.pathname)) return;
+      if (heat && heat(req, res, url)) return;
 
       if (method === 'GET' && url.pathname === '/a/health') return send(res, 200, { ok: true });
 
@@ -80,6 +85,7 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged
             db.prepare('DELETE FROM events_raw WHERE site = ?').run(key);
             db.prepare('DELETE FROM clicks WHERE site = ?').run(key);
             db.prepare('DELETE FROM daily WHERE site = ?').run(key);
+            db.prepare('DELETE FROM agg_state WHERE site = ?').run(key);
             db.exec('COMMIT');
           } catch (e) {
             db.exec('ROLLBACK');
@@ -100,6 +106,16 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged
         onSitesChanged?.();
         return send(res, 200, { ok: true });
       }
+
+      if (role === 'admin' && method === 'GET' && parts[0] === 'a' && parts[1] === 'admin' && parts[2] === 'stats' && parts.length === 4) {
+        if (!tokenOk(req.headers.authorization, config.adminToken)) return send(res, 401, { error: 'Nicht autorisiert' });
+        const key = parts[3];
+        if (!SITE_KEY.test(key)) return send(res, 400, { error: 'Ungültiger Website-Schlüssel' });
+        const r = url.searchParams.get('range') ?? '30';
+        if (!['7', '30', '90'].includes(r)) return send(res, 400, { error: 'Ungültiger Zeitraum (7, 30 oder 90)' });
+        if (!db.prepare('SELECT 1 FROM sites WHERE siteKey = ?').get(key)) return send(res, 404, { error: 'Website nicht gefunden' });
+        return send(res, 200, computeStats(db, key, Number(r), clock()));
+      }
       return send(res, 404, { error: 'Nicht gefunden' });
     } catch (e) {
       const status = e.status || 500;
@@ -115,4 +131,4 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged
 export const createPublicServer = ({ config, db, geo, salt, now }) => createHandlerServer({ config, db, role: 'public', geo, salt, now });
 // Interner Listener für die Website-Verwaltung.
 // onSitesChanged: wird nach jeder Änderung an Websites gerufen (public.invalidateSites()).
-export const createAdminServer = ({ config, db, onSitesChanged }) => createHandlerServer({ config, db, role: 'admin', onSitesChanged });
+export const createAdminServer = ({ config, db, onSitesChanged, now }) => createHandlerServer({ config, db, role: 'admin', onSitesChanged, now });

analytics/src/stats.js

@@ -0,0 +1,69 @@
+// Statistik für das Portal: Vergangenheit aus `daily`, noch nicht verdichtete Tage live aus den Rohdaten.
+import { dayOf, aggregateRows } from './jobs.js';
+
+const DAY_MS = 86400 * 1000;
+const TOP = 20;
+
+// Liste der Kalendertage (YYYY-MM-DD), oldest first, endend am Berlin-Tag von `now`.
+function dayList(now, range) {
+  const [y, m, d] = dayOf(now).split('-').map(Number);
+  const days = [];
+  for (let i = range - 1; i >= 0; i--) days.push(new Date(Date.UTC(y, m - 1, d - i)).toISOString().slice(0, 10));
+  return days;
+}
+
+const top = (map) => [...map].map(([key, count]) => ({ key, count })).filter((e) => e.count > 0)
+  .sort((a, b) => b.count - a.count || (a.key < b.key ? -1 : 1)).slice(0, TOP);
+
+export function computeStats(db, site, range, now = Date.now()) {
+  const days = dayList(now, range);
+  const first = days[0];
+  const inRange = new Set(days);
+  const pvByDay = new Map(days.map((d) => [d, 0]));
+  const dims = {};
+  const bump = (dim, key, n) => {
+    if (!dims[dim]) dims[dim] = new Map();
+    dims[dim].set(key, (dims[dim].get(key) || 0) + n);
+  };
+  const apply = (day, dim, key, n) => {
+    if (dim === 'pv_total') pvByDay.set(day, pvByDay.get(day) + n);
+    bump(dim, key, n);
+  };
+
+  const done = new Set(db.prepare('SELECT day FROM agg_state WHERE site = ? AND day >= ?').all(site, first).map((r) => r.day));
+  for (const r of db.prepare('SELECT day, dim, key, count FROM daily WHERE site = ? AND day >= ?').all(site, first)) {
+    if (inRange.has(r.day)) apply(r.day, r.dim, r.key, r.count);
+  }
+  // Tage ohne agg_state (heute, ggf. gestern vor 02:30) live aus den Rohdaten
+  const live = new Map();
+  const from = now - (range + 1) * DAY_MS;
+  for (const r of db.prepare('SELECT ts, type, path, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts >= ? AND ts <= ? ORDER BY ts')
+    .all(site, from, now)) {
+    const d = dayOf(r.ts);
+    if (!inRange.has(d) || done.has(d)) continue;
+    if (!live.has(d)) live.set(d, []);
+    live.get(d).push(r);
+  }
+  for (const [day, rows] of live) for (const [dim, key, n] of aggregateRows(rows)) apply(day, dim, key, n);
+
+  const sum = (dim) => [...(dims[dim] || new Map()).values()].reduce((a, b) => a + b, 0);
+  const banner = { shown: sum('bn_shown'), accept: sum('bn_accept'), reject: sum('bn_reject') };
+  const decided = banner.accept + banner.reject;
+  const contacts = top(dims.ct_kind || new Map());
+  return {
+    totals: {
+      pv: sum('pv_total'),
+      // Summe der Tages-Besucher (kein tagesübergreifender Abgleich möglich: Salz wechselt täglich)
+      visitors: sum('visitors'),
+      contacts: sum('ct_kind'),
+      consentRate: decided ? banner.accept / decided : 0,
+    },
+    series: days.map((day) => ({ day, pv: pvByDay.get(day) })),
+    pages: top(dims.pv_path || new Map()),
+    refs: top(dims.pv_ref || new Map()),
+    countries: top(dims.pv_country || new Map()),
+    devices: top(dims.pv_device || new Map()),
+    contacts,
+    banner,
+  };
+}

analytics/test/heat.test.js

@@ -0,0 +1,102 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { openDb } from '../src/db.js';
+import { createPublicServer, createAdminServer } from '../src/server.js';
+import { signHeatToken, verifyHeatToken } from '../src/heat.js';
+
+const KEY = 'abcdef0123456789abcd';
+const OTHER = 'zzzzzzzz0123456789zz';
+const SECRET = 'geheim';
+const NOW = Date.UTC(2026, 9, 10, 10, 0, 0);
+const exp = (s = 600) => Math.floor(NOW / 1000) + s;
+
+async function setup() {
+  const db = openDb(':memory:');
+  const site = db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)');
+  site.run(KEY, 'p', JSON.stringify(['kunde.de', 'www.kunde.de']));
+  site.run(OTHER, 'q', JSON.stringify(['andere.de']));
+  const click = db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,?,?)');
+  click.run(KEY, NOW - 1000, '/', 'd', 0.1, 0.2);
+  click.run(KEY, NOW - 2000, '/', 'd', 0.3, 0.4);
+  click.run(KEY, NOW - 1000, '/', 'm', 0.9, 0.9);
+  click.run(KEY, NOW - 1000, '/x', 'd', 0.8, 0.8);
+  click.run(OTHER, NOW - 1000, '/', 'd', 0.7, 0.7);
+  click.run(KEY, NOW - 31 * 86400000, '/', 'd', 0.5, 0.5);
+  const server = createPublicServer({ config: { heatSecret: SECRET }, db, now: () => NOW });
+  await new Promise((r) => server.listen(0, '127.0.0.1', r));
+  const base = `http://127.0.0.1:${server.address().port}`;
+  const get = (token, headers = {}) => fetch(`${base}/a/heat?token=${encodeURIComponent(token)}`, { headers });
+  return { db, get, base, close: () => server.close() };
+}
+const tok = (o = {}, secret = SECRET) => signHeatToken({ siteKey: KEY, path: '/', vp: 'd', exp: exp(), ...o }, secret);
+
+test('gültiges Token: neueste Punkte der Seite/Viewport, 30 Tage, no-store, CORS nur eigene Domain', async () => {
+  const s = await setup();
+  const r = await s.get(tok(), { Origin: 'https://www.kunde.de' });
+  assert.equal(r.status, 200);
+  assert.equal(r.headers.get('cache-control'), 'no-store');
+  assert.equal(r.headers.get('access-control-allow-origin'), 'https://www.kunde.de');
+  assert.deepEqual(await r.json(), { points: [{ x: 0.1, y: 0.2 }, { x: 0.3, y: 0.4 }] });
+  const fremd = await s.get(tok(), { Origin: 'https://evil.example' });
+  assert.equal(fremd.status, 200);
+  assert.equal(fremd.headers.get('access-control-allow-origin'), null);
+  const http = await s.get(tok(), { Origin: 'http://kunde.de' });
+  assert.equal(http.headers.get('access-control-allow-origin'), null);
+  assert.equal(r.headers.get('access-control-allow-origin') === '*', false);
+  s.close();
+});
+
+test('maximal 5000 neueste Punkte', async () => {
+  const s = await setup();
+  s.db.exec('BEGIN');
+  const ins = s.db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,?,?)');
+  for (let i = 0; i < 5100; i++) ins.run(KEY, NOW - 10000 + i, '/many', 'd', 0.5, i < 100 ? 0.01 : 0.99);
+  s.db.exec('COMMIT');
+  const j = await (await s.get(tok({ path: '/many' }))).json();
+  assert.equal(j.points.length, 5000);
+  assert.ok(j.points.every((p) => p.y === 0.99)); // die 100 ältesten fehlen
+  s.close();
+});
+
+test('403: abgelaufen, > 30 Min voraus, falsches Secret, manipuliert, Müll, fremde/unbekannte Website', async () => {
+  const s = await setup();
+  const bad = [
+    tok({ exp: exp(-1) }),
+    tok({ exp: exp(30 * 60 + 5) }),
+    tok({}, 'anderes-secret'),
+    'abc', '', 'a.b', 'x'.repeat(10) + '.' + 'f'.repeat(64),
+    tok({ siteKey: 'nichtvorhanden0123456' }),
+  ];
+  // Manipulation: Payload tauschen, Signatur behalten
+  const [part, sig] = tok().split('.');
+  for (const patch of [{ p: '/x' }, { v: 'm' }, { s: OTHER }]) {
+    const o = { ...JSON.parse(Buffer.from(part, 'base64url')), ...patch };
+    bad.push(`${Buffer.from(JSON.stringify(o)).toString('base64url')}.${sig}`);
+  }
+  for (const t of bad) assert.equal((await s.get(t)).status, 403, t.slice(0, 30));
+  assert.equal((await fetch(`${s.base}/a/heat`)).status, 403);
+  // Token der anderen Website liefert nur deren Punkte (nie die von KEY)
+  const j = await (await s.get(tok({ siteKey: OTHER }))).json();
+  assert.deepEqual(j.points, [{ x: 0.7, y: 0.7 }]);
+  s.close();
+});
+
+test('Token mit exp genau 30 Min voraus ist gültig; Helfer verify', () => {
+  assert.ok(verifyHeatToken(tok({ exp: exp(1800) }), SECRET, NOW));
+  assert.equal(verifyHeatToken(tok({ exp: exp(1801) }), SECRET, NOW), null);
+  assert.equal(verifyHeatToken(tok(), '', NOW), null); // leeres Secret sperrt
+  assert.equal(verifyHeatToken(tok({ vp: 'x' }), SECRET, NOW), null);
+  assert.equal(verifyHeatToken(tok({ path: 'kein-slash' }), SECRET, NOW), null);
+  const t = tok();
+  assert.match(t, /^[A-Za-z0-9_-]+\.[0-9a-f]{64}$/);
+  assert.deepEqual(JSON.parse(Buffer.from(t.split('.')[0], 'base64url')), { s: KEY, p: '/', v: 'd', e: exp() });
+});
+
+test('/a/heat nur auf dem öffentlichen Listener', async () => {
+  const db = openDb(':memory:');
+  const server = createAdminServer({ config: { heatSecret: SECRET, adminToken: 't' }, db });
+  await new Promise((r) => server.listen(0, '127.0.0.1', r));
+  const r = await fetch(`http://127.0.0.1:${server.address().port}/a/heat?token=${tok()}`);
+  assert.equal(r.status, 404);
+  server.close();
+});

analytics/test/stats.test.js

@@ -0,0 +1,179 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { openDb } from '../src/db.js';
+import { runNightly, nextRunAt, dayOf } from '../src/jobs.js';
+import { createAdminServer } from '../src/server.js';
+
+const KEY = 'abcdef0123456789abcd';
+const OTHER = 'zzzzzzzz0123456789zz';
+const NOW = Date.UTC(2026, 9, 10, 10, 0, 0); // 10.10.2026 12:00 Berlin
+const H = 3600 * 1000;
+const D = 24 * H;
+
+function mk() {
+  const db = openDb(':memory:');
+  db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(KEY, 'p', '["a.de"]');
+  const ev = (o) => db.prepare(`INSERT INTO events_raw (site, ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind, host)
+    VALUES (@site,@ts,@type,@path,@vp,@ref,@country,@device,@visitor,@sec,@scroll,@kind,@host)`)
+    .run({ site: KEY, path: null, vp: null, ref: null, country: null, device: null, visitor: null, sec: null, scroll: null, kind: null, host: null, ...o });
+  return { db, ev };
+}
+const daily = (db, site = KEY) => db.prepare('SELECT day, dim, key, count FROM daily WHERE site = ? ORDER BY day, dim, key').all(site);
+const get = (db, day, dim, key = '') => db.prepare('SELECT count FROM daily WHERE site=? AND day=? AND dim=? AND key=?').get(KEY, day, dim, key)?.count;
+
+test('Verdichtung: Dimensionen, nur Tage < heute, idempotent', () => {
+  const { db, ev } = mk();
+  const t = NOW - D; // 9.10.
+  ev({ ts: t, type: 'pv', path: '/', vp: 'd', ref: 'google.com', country: 'DE', device: 'd' });
+  ev({ ts: t + 1, type: 'pv', path: '/', vp: 'm', ref: null, country: 'DE', device: 'm' });
+  ev({ ts: t + 2, type: 'pv', path: '/x', vp: 'd', ref: 'google.com', country: 'AT', device: 'd' });
+  ev({ ts: t + 3, type: 'en', path: '/', vp: 'd', visitor: 'v1', sec: 10, scroll: 50 });
+  ev({ ts: t + 4, type: 'en', path: '/x', vp: 'd', visitor: 'v1', sec: 20, scroll: 70 });
+  ev({ ts: t + 5, type: 'ct', path: '/', visitor: 'v2', kind: 'tel' });
+  ev({ ts: t + 6, type: 'ct', path: '/', visitor: 'v2', kind: 'out', host: 'x.de' });
+  ev({ ts: t + 7, type: 'bn', kind: 'shown' });
+  ev({ ts: t + 8, type: 'bn', kind: 'accept' });
+  ev({ ts: NOW, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); // heute: unberührt
+  runNightly(db, { now: NOW });
+  const d = '2026-10-09';
+  assert.equal(get(db, d, 'pv_total'), 3);
+  assert.equal(get(db, d, 'pv_path', '/'), 2);
+  assert.equal(get(db, d, 'pv_path', '/x'), 1);
+  assert.equal(get(db, d, 'pv_ref', 'google.com'), 2);
+  assert.equal(get(db, d, 'pv_ref', ''), 1);
+  assert.equal(get(db, d, 'pv_country', 'DE'), 2);
+  assert.equal(get(db, d, 'pv_device', 'm'), 1);
+  assert.equal(get(db, d, 'visitors'), 2); // v1, v2
+  assert.equal(get(db, d, 'en_count'), 2);
+  assert.equal(get(db, d, 'en_sec_sum'), 30);
+  assert.equal(get(db, d, 'scroll_sum'), 120);
+  assert.equal(get(db, d, 'ct_kind', 'tel'), 1);
+  assert.equal(get(db, d, 'ct_kind', 'out'), 1);
+  assert.equal(get(db, d, 'bn_shown'), 1);
+  assert.equal(get(db, d, 'bn_accept'), 1);
+  assert.equal(get(db, d, 'bn_reject'), undefined);
+  assert.equal(db.prepare("SELECT COUNT(*) c FROM daily WHERE day = '2026-10-10'").get().c, 0);
+  const before = daily(db);
+  runNightly(db, { now: NOW });
+  runNightly(db, { now: NOW + H });
+  assert.deepEqual(daily(db), before);
+});
+
+test('DST: 23:30 UTC im Sommer gehört zum nächsten Berlin-Tag', () => {
+  const { db, ev } = mk();
+  ev({ ts: Date.UTC(2026, 6, 14, 23, 30), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); // 15.7. 01:30 Berlin
+  ev({ ts: Date.UTC(2026, 6, 14, 21, 59), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); // 14.7. 23:59 Berlin
+  assert.equal(dayOf(Date.UTC(2026, 6, 14, 23, 30)), '2026-07-15');
+  runNightly(db, { now: Date.UTC(2026, 6, 16, 10) });
+  assert.equal(get(db, '2026-07-14', 'pv_total'), 1);
+  assert.equal(get(db, '2026-07-15', 'pv_total'), 1);
+  // Winter: 23:30 UTC = 00:30 Berlin am Folgetag
+  assert.equal(dayOf(Date.UTC(2026, 0, 14, 23, 30)), '2026-01-15');
+});
+
+test('Aufbewahrung: Rohdaten und Klicks > 30 Tage weg, vorher verdichtet', () => {
+  const { db, ev } = mk();
+  ev({ ts: NOW - 31 * D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
+  ev({ ts: NOW - 29 * D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
+  const ins = db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,0.5,0.5)');
+  ins.run(KEY, NOW - 31 * D, '/', 'd'); ins.run(KEY, NOW - 29 * D, '/', 'd');
+  runNightly(db, { now: NOW });
+  assert.equal(db.prepare('SELECT COUNT(*) c FROM events_raw').get().c, 1);
+  assert.equal(db.prepare('SELECT COUNT(*) c FROM clicks').get().c, 1);
+  assert.equal(get(db, dayOf(NOW - 31 * D), 'pv_total'), 1); // in daily erhalten
+});
+
+test('Website mit offSince > 60 Tage wird komplett gelöscht', () => {
+  const { db, ev } = mk();
+  db.prepare('INSERT INTO sites (siteKey, projectId, domains, active, offSince) VALUES (?,?,?,0,?)')
+    .run(OTHER, 'q', '["b.de"]', new Date(NOW - 61 * D).toISOString());
+  db.prepare('INSERT INTO sites (siteKey, projectId, domains, active, offSince) VALUES (?,?,?,0,?)')
+    .run('young0123456789abcdef', 'r', '["c.de"]', new Date(NOW - 59 * D).toISOString());
+  ev({ site: OTHER, ts: NOW - D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
+  ev({ ts: NOW - D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
+  db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,0.1,0.1)').run(OTHER, NOW - D, '/', 'd');
+  runNightly(db, { now: NOW });
+  for (const t of ['sites', 'events_raw', 'clicks', 'daily', 'agg_state']) {
+    const col = t === 'sites' ? 'siteKey' : 'site';
+    assert.equal(db.prepare(`SELECT COUNT(*) c FROM ${t} WHERE ${col} = ?`).get(OTHER).c, 0, t);
+  }
+  assert.equal(db.prepare('SELECT COUNT(*) c FROM sites').get().c, 2); // KEY + junge
+  assert.ok(get(db, dayOf(NOW - D), 'pv_total'));
+});
+
+test('nextRunAt: 02:30 Berlin (Sommer/Winter)', () => {
+  assert.equal(new Date(nextRunAt(Date.UTC(2026, 9, 1, 10))).toISOString(), '2026-10-02T00:30:00.000Z');
+  assert.equal(new Date(nextRunAt(Date.UTC(2026, 9, 1, 0, 0))).toISOString(), '2026-10-01T00:30:00.000Z');
+  assert.equal(new Date(nextRunAt(Date.UTC(2026, 0, 1, 12))).toISOString(), '2026-01-02T01:30:00.000Z');
+});
+
+async function admin(db, now = () => NOW) {
+  const server = createAdminServer({ config: { adminToken: 'tok' }, db, now });
+  await new Promise((r) => server.listen(0, '127.0.0.1', r));
+  const base = `http://127.0.0.1:${server.address().port}`;
+  const get = (p, auth = 'Bearer tok') => fetch(base + p, { headers: auth ? { Authorization: auth } : {} });
+  return { get, close: () => server.close() };
+}
+
+test('Stats-Endpunkt: Form, Serie lückenlos, heute live + Vergangenheit aus daily, consentRate', async () => {
+  const { db, ev } = mk();
+  const y = NOW - D;
+  for (let i = 0; i < 3; i++) ev({ ts: y + i, type: 'pv', path: '/a', vp: 'd', ref: 'g.de', country: 'DE', device: 'd' });
+  ev({ ts: y + 10, type: 'en', path: '/a', vp: 'd', visitor: 'v1', sec: 5, scroll: 5 });
+  ev({ ts: y + 11, type: 'bn', kind: 'accept' });
+  runNightly(db, { now: NOW }); // gestern -> daily
+  ev({ ts: NOW - H, type: 'pv', path: '/b', vp: 'm', country: 'AT', device: 'm' }); // heute live
+  ev({ ts: NOW - H, type: 'ct', path: '/b', visitor: 'v9', kind: 'mail' });
+  ev({ ts: NOW - H, type: 'bn', kind: 'accept' });
+  ev({ ts: NOW - H, type: 'bn', kind: 'reject' });
+  ev({ ts: NOW - H, type: 'bn', kind: 'reject' });
+  ev({ ts: NOW - H, type: 'bn', kind: 'shown' });
+  const a = await admin(db);
+  const r = await a.get(`/a/admin/stats/${KEY}?range=7`);
+  assert.equal(r.status, 200);
+  const j = await r.json();
+  assert.deepEqual(Object.keys(j).sort(), ['banner', 'contacts', 'countries', 'devices', 'pages', 'refs', 'series', 'totals']);
+  assert.equal(j.series.length, 7);
+  assert.equal(j.series[0].day, '2026-10-04');
+  assert.equal(j.series[6].day, '2026-10-10');
+  assert.deepEqual(j.series.map((s) => s.pv), [0, 0, 0, 0, 0, 3, 1]);
+  assert.deepEqual(j.totals, { pv: 4, visitors: 2, contacts: 1, consentRate: 2 / 4 });
+  assert.deepEqual(j.pages, [{ key: '/a', count: 3 }, { key: '/b', count: 1 }]);
+  assert.deepEqual(j.refs[0], { key: 'g.de', count: 3 });
+  assert.deepEqual(j.countries, [{ key: 'DE', count: 3 }, { key: 'AT', count: 1 }]);
+  assert.deepEqual(j.devices, [{ key: 'd', count: 3 }, { key: 'm', count: 1 }]);
+  assert.deepEqual(j.contacts, [{ key: 'mail', count: 1 }]);
+  assert.deepEqual(j.banner, { shown: 1, accept: 2, reject: 2 });
+  // range 90: 90 Tage
+  const j90 = await (await a.get(`/a/admin/stats/${KEY}?range=90`)).json();
+  assert.equal(j90.series.length, 90);
+  assert.equal(j90.totals.pv, 4);
+  a.close();
+});
+
+test('Stats: gestern noch nicht verdichtet wird live gezählt, nie doppelt', async () => {
+  const { db, ev } = mk();
+  ev({ ts: NOW - D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
+  const a = await admin(db);
+  assert.equal((await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json()).totals.pv, 1);
+  runNightly(db, { now: NOW });
+  assert.equal((await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json()).totals.pv, 1);
+  a.close();
+});
+
+test('Stats: Top 20 absteigend, consentRate 0 ohne Entscheidungen, Fehlerfälle', async () => {
+  const { db, ev } = mk();
+  for (let p = 0; p < 25; p++) for (let i = 0; i <= p; i++) ev({ ts: NOW - 1000, type: 'pv', path: `/p${p}`, vp: 'd', country: 'DE', device: 'd' });
+  const a = await admin(db);
+  const j = await (await a.get(`/a/admin/stats/${KEY}?range=30`)).json();
+  assert.equal(j.pages.length, 20);
+  assert.equal(j.pages[0].key, '/p24');
+  assert.ok(j.pages.every((e, i, arr) => !i || arr[i - 1].count >= e.count));
+  assert.equal(j.totals.consentRate, 0);
+  assert.equal((await a.get(`/a/admin/stats/${KEY}?range=5`)).status, 400);
+  assert.equal((await a.get(`/a/admin/stats/${KEY}?range=abc`)).status, 400);
+  assert.equal((await a.get(`/a/admin/stats/${OTHER}?range=7`)).status, 404);
+  assert.equal((await a.get(`/a/admin/stats/${KEY}?range=7`, null)).status, 401);
+  assert.equal((await a.get(`/a/admin/stats/${KEY}?range=7`, 'Bearer nope')).status, 401);
+  a.close();
+});