Analytics: Verdichtung, Statistik, Heatmap-Daten

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

analytics/src/jobs.js

@@ -0,0 +1,119 @@
+// Nachtjob: Rohdaten tageweise in `daily` verdichten, Fristen durchsetzen.
+// Idempotenz: agg_state(site, day) merkt sich verdichtete Tage; ein Tag wird nie zweimal gezählt.
+// Rohzeilen bleiben bis zur 30-Tage-Löschung bestehen (Statistik für heute/gestern liest sie live).
+const DAY_FMT = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Berlin', year: 'numeric', month: '2-digit', day: '2-digit' });
+const WALL_FMT = new Intl.DateTimeFormat('en-GB', {
+  timeZone: 'Europe/Berlin', hourCycle: 'h23', year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit',
+});
+const DAY_MS = 86400 * 1000;
+export const RAW_DAYS = 30;
+export const OFF_DAYS = 60;
+
+// Kalendertag (YYYY-MM-DD) in Europe/Berlin.
+export const dayOf = (ts) => DAY_FMT.format(ts);
+
+// Verdichtet Rohzeilen EINES Tages zu [dim, key, count]-Tripeln.
+export function aggregateRows(rows) {
+  const m = new Map();
+  const add = (dim, key, n = 1) => { const k = `${dim}\u0000${key}`; m.set(k, (m.get(k) || 0) + n); };
+  const visitors = new Set();
+  for (const r of rows) {
+    if (r.type === 'pv') {
+      add('pv_total', ''); add('pv_path', r.path || ''); add('pv_ref', r.ref || '');
+      add('pv_country', r.country || ''); add('pv_device', r.device || '');
+    } else if (r.type === 'en') {
+      add('en_count', ''); add('en_sec_sum', '', r.sec || 0); add('scroll_sum', '', r.scroll || 0);
+      if (r.visitor) visitors.add(r.visitor);
+    } else if (r.type === 'ct') {
+      add('ct_kind', r.kind || '');
+      if (r.visitor) visitors.add(r.visitor);
+    } else if (r.type === 'bn' && ['shown', 'accept', 'reject'].includes(r.kind)) {
+      add(`bn_${r.kind}`, '');
+    }
+  }
+  if (visitors.size) add('visitors', '', visitors.size);
+  return [...m].filter(([, n]) => n > 0).map(([k, n]) => { const [dim, key] = k.split('\u0000'); return [dim, key, n]; });
+}
+
+export function runNightly(db, { now = Date.now() } = {}) {
+  const today = dayOf(now);
+  const rawCut = now - RAW_DAYS * DAY_MS;
+  const offCut = new Date(now - OFF_DAYS * DAY_MS).toISOString();
+  const siteKeys = db.prepare(`SELECT DISTINCT site FROM events_raw UNION SELECT siteKey FROM sites`).all().map((r) => r.site);
+  const selRaw = db.prepare('SELECT ts, type, path, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts < ? ORDER BY ts');
+  const selDone = db.prepare('SELECT day FROM agg_state WHERE site = ?');
+  const insDaily = db.prepare(`INSERT INTO daily (site, day, dim, key, count) VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(site, day, dim, key) DO UPDATE SET count = count + excluded.count`);
+  const insDone = db.prepare('INSERT OR IGNORE INTO agg_state (site, day) VALUES (?, ?)');
+  const stats = { aggregated: 0, sitesDeleted: 0 };
+
+  const tx = (fn) => {
+    db.exec('BEGIN');
+    try { fn(); db.exec('COMMIT'); } catch (e) { db.exec('ROLLBACK'); throw e; }
+  };
+
+  for (const site of siteKeys) {
+    tx(() => {
+      const done = new Set(selDone.all(site).map((r) => r.day));
+      const byDay = new Map();
+      for (const r of selRaw.all(site, now)) {
+        const d = dayOf(r.ts);
+        if (d >= today || done.has(d)) continue;
+        if (!byDay.has(d)) byDay.set(d, []);
+        byDay.get(d).push(r);
+      }
+      for (const [day, rows] of byDay) {
+        for (const [dim, key, n] of aggregateRows(rows)) insDaily.run(site, day, dim, key, n);
+        insDone.run(site, day);
+        stats.aggregated++;
+      }
+      // Fristen erst NACH der Verdichtung (Rohdaten und Klicks > 30 Tage)
+      db.prepare('DELETE FROM events_raw WHERE site = ? AND ts < ?').run(site, rawCut);
+      db.prepare('DELETE FROM clicks WHERE site = ? AND ts < ?').run(site, rawCut);
+    });
+  }
+  // Klicks ganz ohne Website-Zeile / Ereignisse
+  db.prepare('DELETE FROM clicks WHERE ts < ?').run(rawCut);
+
+  for (const { siteKey } of db.prepare('SELECT siteKey FROM sites WHERE offSince IS NOT NULL AND offSince < ?').all(offCut)) {
+    tx(() => {
+      for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) db.prepare(`DELETE FROM ${t} WHERE site = ?`).run(siteKey);
+      db.prepare('DELETE FROM sites WHERE siteKey = ?').run(siteKey);
+      stats.sitesDeleted++;
+    });
+  }
+  return stats;
+}
+
+// Offset (ms) von Europe/Berlin gegenüber UTC zum Zeitpunkt ts.
+function berlinOffset(ts) {
+  const p = Object.fromEntries(WALL_FMT.formatToParts(ts).map((x) => [x.type, x.value]));
+  return Date.UTC(p.year, p.month - 1, p.day, p.hour, p.minute, p.second) - Math.floor(ts / 1000) * 1000;
+}
+
+// Nächster Lauf um hour:minute Europe/Berlin (strikt nach now), als UTC-ms.
+export function nextRunAt(now, hour = 2, minute = 30) {
+  const [y, m, d] = dayOf(now).split('-').map(Number);
+  const at = (dd) => {
+    const guess = Date.UTC(y, m - 1, dd, hour, minute);
+    return guess - berlinOffset(guess - berlinOffset(guess));
+  };
+  const t = at(d);
+  return t > now ? t : at(d + 1);
+}
+
+// Plant runNightly täglich 02:30 Berlin und einmal kurz nach dem Start; stop() beendet alle Timer.
+export function scheduleNightly(db, { startDelayMs = 60000, log = console } = {}) {
+  let timer = null;
+  let stopped = false;
+  const run = () => {
+    try {
+      const r = runNightly(db, { now: Date.now() });
+      log.log(`Analytics-Nachtjob: ${r.aggregated} Tage verdichtet, ${r.sitesDeleted} Websites gelöscht`);
+    } catch (e) { log.error('Analytics-Nachtjob:', e.message); }
+  };
+  const arm = (delay, fn) => { if (!stopped) timer = setTimeout(fn, delay); };
+  const daily = () => { run(); arm(nextRunAt(Date.now()) - Date.now(), daily); };
+  arm(startDelayMs, () => { run(); arm(nextRunAt(Date.now()) - Date.now(), daily); });
+  return { stop: () => { stopped = true; clearTimeout(timer); } };
+}