// Nachtjob: Rohdaten tageweise in `daily` verdichten, Fristen durchsetzen. // Idempotenz: agg_state(site, day) merkt sich verdichtete Tage; ein Tag wird nie zweimal gezählt. // Rohzeilen bleiben bis zur 30-Tage-Löschung bestehen (Statistik für heute/gestern liest sie live). const DAY_FMT = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Berlin', year: 'numeric', month: '2-digit', day: '2-digit' }); const WALL_FMT = new Intl.DateTimeFormat('en-GB', { timeZone: 'Europe/Berlin', hourCycle: 'h23', year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit', }); const DAY_MS = 86400 * 1000; export const RAW_DAYS = 30; export const OFF_DAYS = 60; export const DAILY_MONTHS = 25; // Berliner Tag von `now` minus DAILY_MONTHS Kalendermonate (Monatsende auf den letzten Tag des Zielmonats begrenzt). export function dailyCutoffDay(now) { const [y, m, d] = DAY_FMT.format(now).split('-').map(Number); const idx = y * 12 + (m - 1) - DAILY_MONTHS; const ty = Math.floor(idx / 12); const tm = idx % 12; const last = new Date(Date.UTC(ty, tm + 1, 0)).getUTCDate(); return `${String(ty).padStart(4, '0')}-${String(tm + 1).padStart(2, '0')}-${String(Math.min(d, last)).padStart(2, '0')}`; } // Kalendertag (YYYY-MM-DD) in Europe/Berlin. export const dayOf = (ts) => DAY_FMT.format(ts); // Verdichtet Rohzeilen EINES Tages zu [dim, key, count]-Tripeln. export function aggregateRows(rows) { const m = new Map(); const add = (dim, key, n = 1) => { const k = `${dim}\u0000${key}`; m.set(k, (m.get(k) || 0) + n); }; const visitors = new Set(); for (const r of rows) { if (r.type === 'pv') { add('pv_total', ''); add('pv_path', r.path || ''); if (r.vp != null) add('pv_ref', r.ref || ''); // nur Besucher mit Einwilligung; sonst kommt pv_ref aus cx add('pv_country', r.country || ''); add('pv_device', r.device || ''); } else if (r.type === 'cx') { add('pv_ref', r.ref || ''); } else if (r.type === 'en') { add('en_count', ''); add('en_sec_sum', '', r.sec || 0); add('scroll_sum', '', r.scroll || 0); if (r.visitor) visitors.add(r.visitor); } else if (r.type === 'ct') { add('ct_kind', r.kind || ''); if (r.visitor) visitors.add(r.visitor); } else if (r.type === 'bn' && ['shown', 'accept', 'reject'].includes(r.kind)) { add(`bn_${r.kind}`, ''); } } if (visitors.size) add('visitors', '', visitors.size); return [...m].filter(([, n]) => n > 0).map(([k, n]) => { const [dim, key] = k.split('\u0000'); return [dim, key, n]; }); } export function runNightly(db, { now = Date.now(), log = console } = {}) { const today = dayOf(now); const rawCut = now - RAW_DAYS * DAY_MS; const offCut = new Date(now - OFF_DAYS * DAY_MS).toISOString(); const siteKeys = db.prepare(`SELECT DISTINCT site FROM events_raw UNION SELECT siteKey FROM sites`).all().map((r) => r.site); const selRaw = db.prepare('SELECT ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts < ? ORDER BY ts'); const selDone = db.prepare('SELECT day FROM agg_state WHERE site = ?'); const insDaily = db.prepare(`INSERT INTO daily (site, day, dim, key, count) VALUES (?, ?, ?, ?, ?) ON CONFLICT(site, day, dim, key) DO UPDATE SET count = count + excluded.count`); const insDone = db.prepare('INSERT OR IGNORE INTO agg_state (site, day) VALUES (?, ?)'); const stats = { aggregated: 0, sitesDeleted: 0, dailyDeleted: 0, orphansMarked: 0, orphanRowsDeleted: 0 }; const tx = (fn) => { db.exec('BEGIN'); try { fn(); db.exec('COMMIT'); } catch (e) { db.exec('ROLLBACK'); throw e; } }; for (const site of siteKeys) { tx(() => { const done = new Set(selDone.all(site).map((r) => r.day)); const byDay = new Map(); for (const r of selRaw.all(site, now)) { const d = dayOf(r.ts); if (d >= today || done.has(d)) continue; if (!byDay.has(d)) byDay.set(d, []); byDay.get(d).push(r); } for (const [day, rows] of byDay) { for (const [dim, key, n] of aggregateRows(rows)) insDaily.run(site, day, dim, key, n); insDone.run(site, day); stats.aggregated++; } // Fristen erst NACH der Verdichtung (Rohdaten und Klicks > 30 Tage) db.prepare('DELETE FROM events_raw WHERE site = ? AND ts < ?').run(site, rawCut); db.prepare('DELETE FROM clicks WHERE site = ? AND ts < ?').run(site, rawCut); }); } // Klicks ganz ohne Website-Zeile / Ereignisse db.prepare('DELETE FROM clicks WHERE ts < ?').run(rawCut); for (const { siteKey } of db.prepare('SELECT siteKey FROM sites WHERE offSince IS NOT NULL AND offSince < ?').all(offCut)) { tx(() => { for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) db.prepare(`DELETE FROM ${t} WHERE site = ?`).run(siteKey); db.prepare('DELETE FROM sites WHERE siteKey = ?').run(siteKey); stats.sitesDeleted++; }); } // Tageswerte älter als 25 Monate (alle Websites) tx(() => { const cut = dailyCutoffDay(now); stats.dailyDeleted = Number(db.prepare('DELETE FROM daily WHERE day < ?').run(cut).changes); db.prepare('DELETE FROM agg_state WHERE day < ?').run(cut); }); // Ausgeschaltete Websites ohne offSince: Frist beginnt bei Entdeckung. stats.orphansMarked = Number(db.prepare('UPDATE sites SET offSince = ? WHERE active = 0 AND offSince IS NULL').run(new Date(now).toISOString()).changes); // Zeilen ohne Website-Eintrag tx(() => { for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) { stats.orphanRowsDeleted += Number(db.prepare(`DELETE FROM ${t} WHERE site NOT IN (SELECT siteKey FROM sites)`).run().changes); } }); try { db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); } catch (e) { log.error('Analytics-Nachtjob: wal_checkpoint:', e.message); } return stats; } // Offset (ms) von Europe/Berlin gegenüber UTC zum Zeitpunkt ts. function berlinOffset(ts) { const p = Object.fromEntries(WALL_FMT.formatToParts(ts).map((x) => [x.type, x.value])); return Date.UTC(p.year, p.month - 1, p.day, p.hour, p.minute, p.second) - Math.floor(ts / 1000) * 1000; } // Nächster Lauf um hour:minute Europe/Berlin (strikt nach now), als UTC-ms. export function nextRunAt(now, hour = 2, minute = 30) { const [y, m, d] = dayOf(now).split('-').map(Number); const at = (dd) => { const guess = Date.UTC(y, m - 1, dd, hour, minute); return guess - berlinOffset(guess - berlinOffset(guess)); }; const t = at(d); return t > now ? t : at(d + 1); } // Plant runNightly täglich 02:30 Berlin und einmal kurz nach dem Start; stop() beendet alle Timer. // Test-Hooks: now, timers ({ setTimeout, clearTimeout }) und job (statt runNightly). export function scheduleNightly(db, { startDelayMs = 60000, log = console, now = Date.now, timers = { setTimeout, clearTimeout }, job = runNightly } = {}) { let timer = null; let stopped = false; const run = () => { try { const r = job(db, { now: now() }); log.log(`Analytics-Nachtjob: ${r.aggregated} Tage verdichtet, ${r.sitesDeleted} Websites gelöscht, ${r.dailyDeleted} Tageswerte > ${DAILY_MONTHS} Monate gelöscht, ${r.orphansMarked} Websites markiert, ${r.orphanRowsDeleted} verwaiste Zeilen gelöscht`); } catch (e) { log.error('Analytics-Nachtjob:', e.message); } }; const arm = (delay, fn) => { if (!stopped) timer = timers.setTimeout(fn, delay); }; // Ein Timer kann früh feuern (Uhrsprung, Timer-Ungenauigkeit): dann nur neu auf den Zielzeitpunkt setzen, // nie vorzeitig laufen. Nach dem Lauf strikt hinter das Ziel planen, damit 02:30 nie doppelt läuft. const planDaily = (target) => arm(Math.max(0, target - now()), () => { if (now() < target) return planDaily(target); run(); planDaily(nextRunAt(Math.max(now(), target) + 1000)); }); arm(startDelayMs, () => { run(); planDaily(nextRunAt(now())); }); return { stop: () => { stopped = true; timers.clearTimeout(timer); } }; }