Analytics: Verdichtung, Statistik, Heatmap-Daten

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

analytics/src/db.js

@@ -44,6 +44,12 @@ CREATE TABLE IF NOT EXISTS daily (
   count INTEGER NOT NULL DEFAULT 0,
   PRIMARY KEY (site, day, dim, key)
 );
+-- Merker für die Nachtverdichtung: Tage, die schon in daily stehen (macht runNightly idempotent).
+CREATE TABLE IF NOT EXISTS agg_state (
+  site TEXT NOT NULL,
+  day TEXT NOT NULL,
+  PRIMARY KEY (site, day)
+);
 `;
 
 export function openDb(path) {

analytics/src/heat.js

@@ -0,0 +1,71 @@
+// GET /a/heat?token= (öffentlicher Listener): Klickpunkte für das Heatmap-Overlay.
+// Token (R3): <b64url(JSON {s,p,v,e})>.<hex HMAC-SHA256(heatSecret, b64part)>, e = Ablauf in Unix-Sekunden.
+import crypto from 'node:crypto';
+import { hostOf } from './ingest.js';
+
+const SITE_KEY = /^[a-z0-9]{16,40}$/;
+const MAX_AHEAD = 30 * 60; // Sekunden
+const POINTS = 5000;
+const CLICK_DAYS = 30;
+
+export function signHeatToken({ siteKey, path, vp, exp }, secret) {
+  const part = Buffer.from(JSON.stringify({ s: siteKey, p: path, v: vp, e: exp })).toString('base64url');
+  return `${part}.${crypto.createHmac('sha256', secret).update(part).digest('hex')}`;
+}
+
+// Liefert { siteKey, path, vp, exp } oder null (Signatur, Format, abgelaufen, > 30 Min voraus).
+export function verifyHeatToken(token, secret, nowMs) {
+  if (!secret || typeof token !== 'string') return null;
+  const m = /^([A-Za-z0-9_-]+)\.([0-9a-f]{64})$/.exec(token);
+  if (!m) return null;
+  const expect = crypto.createHmac('sha256', secret).update(m[1]).digest();
+  const got = Buffer.from(m[2], 'hex');
+  if (got.length !== expect.length || !crypto.timingSafeEqual(got, expect)) return null;
+  let o;
+  try { o = JSON.parse(Buffer.from(m[1], 'base64url').toString('utf8')); } catch { return null; }
+  if (!o || typeof o !== 'object' || typeof o.s !== 'string' || !SITE_KEY.test(o.s)
+    || typeof o.p !== 'string' || !o.p.startsWith('/') || !['m', 't', 'd'].includes(o.v)
+    || !Number.isInteger(o.e)) return null;
+  const nowS = nowMs / 1000;
+  if (o.e <= nowS || o.e > nowS + MAX_AHEAD) return null;
+  return { siteKey: o.s, path: o.p, vp: o.v, exp: o.e };
+}
+
+export function createHeat({ db, config = {}, now }) {
+  const clock = now || Date.now;
+  const getSite = db.prepare('SELECT domains FROM sites WHERE siteKey = ?');
+  const getClicks = db.prepare('SELECT x, y FROM clicks WHERE site = ? AND path = ? AND vp = ? AND ts >= ? ORDER BY ts DESC LIMIT ?');
+
+  // CORS: nur die eigenen https-Domains der Website (das Overlay läuft auf der Kundenseite).
+  function cors(res, req, domains) {
+    const origin = req.headers.origin;
+    const host = hostOf(origin);
+    res.setHeader('Vary', 'Origin');
+    if (host && domains.includes(host)) res.setHeader('Access-Control-Allow-Origin', origin);
+  }
+  const domainsOf = (site) => { try { return JSON.parse(site.domains); } catch { return []; } };
+
+  // true, wenn die Anfrage /a/heat betraf.
+  return function handle(req, res, url) {
+    if (url.pathname !== '/a/heat') return false;
+    const end = (status, body, extra = {}) => {
+      const b = body ? JSON.stringify(body) : '';
+      res.writeHead(status, { 'Cache-Control': 'no-store', ...(body ? { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(b) } : {}), ...extra });
+      res.end(b);
+      return true;
+    };
+    if (req.method === 'OPTIONS') {
+      // Preflight nur bei eigener Domain mit CORS-Köpfen beantworten; Token steht in der URL, Daten gibt es erst beim GET.
+      res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
+      return end(204);
+    }
+    if (req.method !== 'GET') return end(405, null, { Allow: 'GET, OPTIONS' });
+    const t = verifyHeatToken(url.searchParams.get('token'), config.heatSecret, clock());
+    const site = t && getSite.get(t.siteKey);
+    if (!t || !site) return end(403, { error: 'Ungültiger oder abgelaufener Link' });
+    cors(res, req, domainsOf(site));
+    const since = clock() - CLICK_DAYS * 86400 * 1000;
+    const points = getClicks.all(t.siteKey, t.path, t.vp, since, POINTS).map((r) => ({ x: r.x, y: r.y }));
+    return end(200, { points });
+  };
+}

analytics/src/index.js

@@ -4,6 +4,7 @@ import { loadConfig } from './config.js';
 import { openDb } from './db.js';
 import { createSalt } from './salt.js';
 import { loadGeo } from './geo.js';
+import { scheduleNightly } from './jobs.js';
 import { createPublicServer, createAdminServer } from './server.js';
 
 const config = loadConfig(process.env);
@@ -22,7 +23,11 @@ for (const [name, server, port] of servers) {
   server.listen(port, () => console.log(`Analytics-Dienst ${name} auf Port ${port}`));
 }
 
+// Nachtjob: täglich 02:30 Europe/Berlin und einmal 60 s nach dem Start
+const nightly = scheduleNightly(db);
+
 process.on('SIGTERM', () => {
+  nightly.stop();
   let open = servers.length;
   const done = () => { if (--open === 0) { db.close(); process.exit(0); } };
   for (const [, server] of servers) { server.close(done); server.closeIdleConnections?.(); }

analytics/src/ingest.js

@@ -42,7 +42,7 @@ function clientIp(req) {
 }
 
 // Nur https ohne expliziten Port zählt; alles andere ergibt '' (kein Treffer).
-function hostOf(value) {
+export function hostOf(value) {
   if (typeof value !== 'string' || !value) return '';
   try {
     const u = new URL(value);

analytics/src/jobs.js

@@ -0,0 +1,119 @@
+// Nachtjob: Rohdaten tageweise in `daily` verdichten, Fristen durchsetzen.
+// Idempotenz: agg_state(site, day) merkt sich verdichtete Tage; ein Tag wird nie zweimal gezählt.
+// Rohzeilen bleiben bis zur 30-Tage-Löschung bestehen (Statistik für heute/gestern liest sie live).
+const DAY_FMT = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Berlin', year: 'numeric', month: '2-digit', day: '2-digit' });
+const WALL_FMT = new Intl.DateTimeFormat('en-GB', {
+  timeZone: 'Europe/Berlin', hourCycle: 'h23', year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit',
+});
+const DAY_MS = 86400 * 1000;
+export const RAW_DAYS = 30;
+export const OFF_DAYS = 60;
+
+// Kalendertag (YYYY-MM-DD) in Europe/Berlin.
+export const dayOf = (ts) => DAY_FMT.format(ts);
+
+// Verdichtet Rohzeilen EINES Tages zu [dim, key, count]-Tripeln.
+export function aggregateRows(rows) {
+  const m = new Map();
+  const add = (dim, key, n = 1) => { const k = `${dim}\u0000${key}`; m.set(k, (m.get(k) || 0) + n); };
+  const visitors = new Set();
+  for (const r of rows) {
+    if (r.type === 'pv') {
+      add('pv_total', ''); add('pv_path', r.path || ''); add('pv_ref', r.ref || '');
+      add('pv_country', r.country || ''); add('pv_device', r.device || '');
+    } else if (r.type === 'en') {
+      add('en_count', ''); add('en_sec_sum', '', r.sec || 0); add('scroll_sum', '', r.scroll || 0);
+      if (r.visitor) visitors.add(r.visitor);
+    } else if (r.type === 'ct') {
+      add('ct_kind', r.kind || '');
+      if (r.visitor) visitors.add(r.visitor);
+    } else if (r.type === 'bn' && ['shown', 'accept', 'reject'].includes(r.kind)) {
+      add(`bn_${r.kind}`, '');
+    }
+  }
+  if (visitors.size) add('visitors', '', visitors.size);
+  return [...m].filter(([, n]) => n > 0).map(([k, n]) => { const [dim, key] = k.split('\u0000'); return [dim, key, n]; });
+}
+
+export function runNightly(db, { now = Date.now() } = {}) {
+  const today = dayOf(now);
+  const rawCut = now - RAW_DAYS * DAY_MS;
+  const offCut = new Date(now - OFF_DAYS * DAY_MS).toISOString();
+  const siteKeys = db.prepare(`SELECT DISTINCT site FROM events_raw UNION SELECT siteKey FROM sites`).all().map((r) => r.site);
+  const selRaw = db.prepare('SELECT ts, type, path, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts < ? ORDER BY ts');
+  const selDone = db.prepare('SELECT day FROM agg_state WHERE site = ?');
+  const insDaily = db.prepare(`INSERT INTO daily (site, day, dim, key, count) VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(site, day, dim, key) DO UPDATE SET count = count + excluded.count`);
+  const insDone = db.prepare('INSERT OR IGNORE INTO agg_state (site, day) VALUES (?, ?)');
+  const stats = { aggregated: 0, sitesDeleted: 0 };
+
+  const tx = (fn) => {
+    db.exec('BEGIN');
+    try { fn(); db.exec('COMMIT'); } catch (e) { db.exec('ROLLBACK'); throw e; }
+  };
+
+  for (const site of siteKeys) {
+    tx(() => {
+      const done = new Set(selDone.all(site).map((r) => r.day));
+      const byDay = new Map();
+      for (const r of selRaw.all(site, now)) {
+        const d = dayOf(r.ts);
+        if (d >= today || done.has(d)) continue;
+        if (!byDay.has(d)) byDay.set(d, []);
+        byDay.get(d).push(r);
+      }
+      for (const [day, rows] of byDay) {
+        for (const [dim, key, n] of aggregateRows(rows)) insDaily.run(site, day, dim, key, n);
+        insDone.run(site, day);
+        stats.aggregated++;
+      }
+      // Fristen erst NACH der Verdichtung (Rohdaten und Klicks > 30 Tage)
+      db.prepare('DELETE FROM events_raw WHERE site = ? AND ts < ?').run(site, rawCut);
+      db.prepare('DELETE FROM clicks WHERE site = ? AND ts < ?').run(site, rawCut);
+    });
+  }
+  // Klicks ganz ohne Website-Zeile / Ereignisse
+  db.prepare('DELETE FROM clicks WHERE ts < ?').run(rawCut);
+
+  for (const { siteKey } of db.prepare('SELECT siteKey FROM sites WHERE offSince IS NOT NULL AND offSince < ?').all(offCut)) {
+    tx(() => {
+      for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) db.prepare(`DELETE FROM ${t} WHERE site = ?`).run(siteKey);
+      db.prepare('DELETE FROM sites WHERE siteKey = ?').run(siteKey);
+      stats.sitesDeleted++;
+    });
+  }
+  return stats;
+}
+
+// Offset (ms) von Europe/Berlin gegenüber UTC zum Zeitpunkt ts.
+function berlinOffset(ts) {
+  const p = Object.fromEntries(WALL_FMT.formatToParts(ts).map((x) => [x.type, x.value]));
+  return Date.UTC(p.year, p.month - 1, p.day, p.hour, p.minute, p.second) - Math.floor(ts / 1000) * 1000;
+}
+
+// Nächster Lauf um hour:minute Europe/Berlin (strikt nach now), als UTC-ms.
+export function nextRunAt(now, hour = 2, minute = 30) {
+  const [y, m, d] = dayOf(now).split('-').map(Number);
+  const at = (dd) => {
+    const guess = Date.UTC(y, m - 1, dd, hour, minute);
+    return guess - berlinOffset(guess - berlinOffset(guess));
+  };
+  const t = at(d);
+  return t > now ? t : at(d + 1);
+}
+
+// Plant runNightly täglich 02:30 Berlin und einmal kurz nach dem Start; stop() beendet alle Timer.
+export function scheduleNightly(db, { startDelayMs = 60000, log = console } = {}) {
+  let timer = null;
+  let stopped = false;
+  const run = () => {
+    try {
+      const r = runNightly(db, { now: Date.now() });
+      log.log(`Analytics-Nachtjob: ${r.aggregated} Tage verdichtet, ${r.sitesDeleted} Websites gelöscht`);
+    } catch (e) { log.error('Analytics-Nachtjob:', e.message); }
+  };
+  const arm = (delay, fn) => { if (!stopped) timer = setTimeout(fn, delay); };
+  const daily = () => { run(); arm(nextRunAt(Date.now()) - Date.now(), daily); };
+  arm(startDelayMs, () => { run(); arm(nextRunAt(Date.now()) - Date.now(), daily); });
+  return { stop: () => { stopped = true; clearTimeout(timer); } };
+}

analytics/src/server.js

@@ -2,6 +2,8 @@ import http from 'node:http';
 import crypto from 'node:crypto';
 import { createIngest } from './ingest.js';
 import { createSalt } from './salt.js';
+import { createHeat } from './heat.js';
+import { computeStats } from './stats.js';
 
 const MAX_ADMIN_BODY = 64 * 1024;
 const SITE_KEY = /^[a-z0-9]{16,40}$/;
@@ -57,6 +59,8 @@ function validateSite(b) {
 
 function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged }) {
   const ingest = role === 'public' ? createIngest({ db, geo, salt: salt || createSalt({ now }), now }) : null;
+  const heat = role === 'public' ? createHeat({ db, config, now }) : null;
+  const clock = now || Date.now;
   const server = http.createServer(async (req, res) => {
     try {
       const url = new URL(req.url, 'http://x');
@@ -64,6 +68,7 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged
       const method = req.method;
 
       if (ingest && await ingest(req, res, url.pathname)) return;
+      if (heat && heat(req, res, url)) return;
 
       if (method === 'GET' && url.pathname === '/a/health') return send(res, 200, { ok: true });
 
@@ -80,6 +85,7 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged
             db.prepare('DELETE FROM events_raw WHERE site = ?').run(key);
             db.prepare('DELETE FROM clicks WHERE site = ?').run(key);
             db.prepare('DELETE FROM daily WHERE site = ?').run(key);
+            db.prepare('DELETE FROM agg_state WHERE site = ?').run(key);
             db.exec('COMMIT');
           } catch (e) {
             db.exec('ROLLBACK');
@@ -100,6 +106,16 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged
         onSitesChanged?.();
         return send(res, 200, { ok: true });
       }
+
+      if (role === 'admin' && method === 'GET' && parts[0] === 'a' && parts[1] === 'admin' && parts[2] === 'stats' && parts.length === 4) {
+        if (!tokenOk(req.headers.authorization, config.adminToken)) return send(res, 401, { error: 'Nicht autorisiert' });
+        const key = parts[3];
+        if (!SITE_KEY.test(key)) return send(res, 400, { error: 'Ungültiger Website-Schlüssel' });
+        const r = url.searchParams.get('range') ?? '30';
+        if (!['7', '30', '90'].includes(r)) return send(res, 400, { error: 'Ungültiger Zeitraum (7, 30 oder 90)' });
+        if (!db.prepare('SELECT 1 FROM sites WHERE siteKey = ?').get(key)) return send(res, 404, { error: 'Website nicht gefunden' });
+        return send(res, 200, computeStats(db, key, Number(r), clock()));
+      }
       return send(res, 404, { error: 'Nicht gefunden' });
     } catch (e) {
       const status = e.status || 500;
@@ -115,4 +131,4 @@ function createHandlerServer({ config, db, role, geo, salt, now, onSitesChanged
 export const createPublicServer = ({ config, db, geo, salt, now }) => createHandlerServer({ config, db, role: 'public', geo, salt, now });
 // Interner Listener für die Website-Verwaltung.
 // onSitesChanged: wird nach jeder Änderung an Websites gerufen (public.invalidateSites()).
-export const createAdminServer = ({ config, db, onSitesChanged }) => createHandlerServer({ config, db, role: 'admin', onSitesChanged });
+export const createAdminServer = ({ config, db, onSitesChanged, now }) => createHandlerServer({ config, db, role: 'admin', onSitesChanged, now });

analytics/src/stats.js

@@ -0,0 +1,69 @@
+// Statistik für das Portal: Vergangenheit aus `daily`, noch nicht verdichtete Tage live aus den Rohdaten.
+import { dayOf, aggregateRows } from './jobs.js';
+
+const DAY_MS = 86400 * 1000;
+const TOP = 20;
+
+// Liste der Kalendertage (YYYY-MM-DD), oldest first, endend am Berlin-Tag von `now`.
+function dayList(now, range) {
+  const [y, m, d] = dayOf(now).split('-').map(Number);
+  const days = [];
+  for (let i = range - 1; i >= 0; i--) days.push(new Date(Date.UTC(y, m - 1, d - i)).toISOString().slice(0, 10));
+  return days;
+}
+
+const top = (map) => [...map].map(([key, count]) => ({ key, count })).filter((e) => e.count > 0)
+  .sort((a, b) => b.count - a.count || (a.key < b.key ? -1 : 1)).slice(0, TOP);
+
+export function computeStats(db, site, range, now = Date.now()) {
+  const days = dayList(now, range);
+  const first = days[0];
+  const inRange = new Set(days);
+  const pvByDay = new Map(days.map((d) => [d, 0]));
+  const dims = {};
+  const bump = (dim, key, n) => {
+    if (!dims[dim]) dims[dim] = new Map();
+    dims[dim].set(key, (dims[dim].get(key) || 0) + n);
+  };
+  const apply = (day, dim, key, n) => {
+    if (dim === 'pv_total') pvByDay.set(day, pvByDay.get(day) + n);
+    bump(dim, key, n);
+  };
+
+  const done = new Set(db.prepare('SELECT day FROM agg_state WHERE site = ? AND day >= ?').all(site, first).map((r) => r.day));
+  for (const r of db.prepare('SELECT day, dim, key, count FROM daily WHERE site = ? AND day >= ?').all(site, first)) {
+    if (inRange.has(r.day)) apply(r.day, r.dim, r.key, r.count);
+  }
+  // Tage ohne agg_state (heute, ggf. gestern vor 02:30) live aus den Rohdaten
+  const live = new Map();
+  const from = now - (range + 1) * DAY_MS;
+  for (const r of db.prepare('SELECT ts, type, path, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts >= ? AND ts <= ? ORDER BY ts')
+    .all(site, from, now)) {
+    const d = dayOf(r.ts);
+    if (!inRange.has(d) || done.has(d)) continue;
+    if (!live.has(d)) live.set(d, []);
+    live.get(d).push(r);
+  }
+  for (const [day, rows] of live) for (const [dim, key, n] of aggregateRows(rows)) apply(day, dim, key, n);
+
+  const sum = (dim) => [...(dims[dim] || new Map()).values()].reduce((a, b) => a + b, 0);
+  const banner = { shown: sum('bn_shown'), accept: sum('bn_accept'), reject: sum('bn_reject') };
+  const decided = banner.accept + banner.reject;
+  const contacts = top(dims.ct_kind || new Map());
+  return {
+    totals: {
+      pv: sum('pv_total'),
+      // Summe der Tages-Besucher (kein tagesübergreifender Abgleich möglich: Salz wechselt täglich)
+      visitors: sum('visitors'),
+      contacts: sum('ct_kind'),
+      consentRate: decided ? banner.accept / decided : 0,
+    },
+    series: days.map((day) => ({ day, pv: pvByDay.get(day) })),
+    pages: top(dims.pv_path || new Map()),
+    refs: top(dims.pv_ref || new Map()),
+    countries: top(dims.pv_country || new Map()),
+    devices: top(dims.pv_device || new Map()),
+    contacts,
+    banner,
+  };
+}