159 lines
7.8 KiB
JavaScript
159 lines
7.8 KiB
JavaScript
// Nachtjob: Rohdaten tageweise in `daily` verdichten, Fristen durchsetzen.
|
|
// Idempotenz: agg_state(site, day) merkt sich verdichtete Tage; ein Tag wird nie zweimal gezählt.
|
|
// Rohzeilen bleiben bis zur 30-Tage-Löschung bestehen (Statistik für heute/gestern liest sie live).
|
|
const DAY_FMT = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Berlin', year: 'numeric', month: '2-digit', day: '2-digit' });
|
|
const WALL_FMT = new Intl.DateTimeFormat('en-GB', {
|
|
timeZone: 'Europe/Berlin', hourCycle: 'h23', year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit',
|
|
});
|
|
const DAY_MS = 86400 * 1000;
|
|
export const RAW_DAYS = 30;
|
|
export const OFF_DAYS = 60;
|
|
export const DAILY_MONTHS = 25;
|
|
|
|
// Berliner Tag von `now` minus DAILY_MONTHS Kalendermonate (Monatsende auf den letzten Tag des Zielmonats begrenzt).
|
|
export function dailyCutoffDay(now) {
|
|
const [y, m, d] = DAY_FMT.format(now).split('-').map(Number);
|
|
const idx = y * 12 + (m - 1) - DAILY_MONTHS;
|
|
const ty = Math.floor(idx / 12);
|
|
const tm = idx % 12;
|
|
const last = new Date(Date.UTC(ty, tm + 1, 0)).getUTCDate();
|
|
return `${String(ty).padStart(4, '0')}-${String(tm + 1).padStart(2, '0')}-${String(Math.min(d, last)).padStart(2, '0')}`;
|
|
}
|
|
|
|
// Kalendertag (YYYY-MM-DD) in Europe/Berlin.
|
|
export const dayOf = (ts) => DAY_FMT.format(ts);
|
|
|
|
// Verdichtet Rohzeilen EINES Tages zu [dim, key, count]-Tripeln.
|
|
export function aggregateRows(rows) {
|
|
const m = new Map();
|
|
const add = (dim, key, n = 1) => { const k = `${dim}\u0000${key}`; m.set(k, (m.get(k) || 0) + n); };
|
|
const visitors = new Set();
|
|
for (const r of rows) {
|
|
if (r.type === 'pv') {
|
|
add('pv_total', ''); add('pv_path', r.path || '');
|
|
if (r.vp != null) add('pv_ref', r.ref || ''); // nur Besucher mit Einwilligung; sonst kommt pv_ref aus cx
|
|
add('pv_country', r.country || ''); add('pv_device', r.device || '');
|
|
} else if (r.type === 'cx') {
|
|
add('pv_ref', r.ref || '');
|
|
} else if (r.type === 'en') {
|
|
add('en_count', ''); add('en_sec_sum', '', r.sec || 0); add('scroll_sum', '', r.scroll || 0);
|
|
if (r.visitor) visitors.add(r.visitor);
|
|
} else if (r.type === 'ct') {
|
|
add('ct_kind', r.kind || '');
|
|
if (r.visitor) visitors.add(r.visitor);
|
|
} else if (r.type === 'bn' && ['shown', 'accept', 'reject'].includes(r.kind)) {
|
|
add(`bn_${r.kind}`, '');
|
|
}
|
|
}
|
|
if (visitors.size) add('visitors', '', visitors.size);
|
|
return [...m].filter(([, n]) => n > 0).map(([k, n]) => { const [dim, key] = k.split('\u0000'); return [dim, key, n]; });
|
|
}
|
|
|
|
export function runNightly(db, { now = Date.now(), log = console } = {}) {
|
|
const today = dayOf(now);
|
|
const rawCut = now - RAW_DAYS * DAY_MS;
|
|
const offCut = new Date(now - OFF_DAYS * DAY_MS).toISOString();
|
|
const siteKeys = db.prepare(`SELECT DISTINCT site FROM events_raw UNION SELECT siteKey FROM sites`).all().map((r) => r.site);
|
|
const selRaw = db.prepare('SELECT ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts < ? ORDER BY ts');
|
|
const selDone = db.prepare('SELECT day FROM agg_state WHERE site = ?');
|
|
const insDaily = db.prepare(`INSERT INTO daily (site, day, dim, key, count) VALUES (?, ?, ?, ?, ?)
|
|
ON CONFLICT(site, day, dim, key) DO UPDATE SET count = count + excluded.count`);
|
|
const insDone = db.prepare('INSERT OR IGNORE INTO agg_state (site, day) VALUES (?, ?)');
|
|
const stats = { aggregated: 0, sitesDeleted: 0, dailyDeleted: 0, orphansMarked: 0, orphanRowsDeleted: 0 };
|
|
|
|
const tx = (fn) => {
|
|
db.exec('BEGIN');
|
|
try { fn(); db.exec('COMMIT'); } catch (e) { db.exec('ROLLBACK'); throw e; }
|
|
};
|
|
|
|
for (const site of siteKeys) {
|
|
tx(() => {
|
|
const done = new Set(selDone.all(site).map((r) => r.day));
|
|
const byDay = new Map();
|
|
for (const r of selRaw.all(site, now)) {
|
|
const d = dayOf(r.ts);
|
|
if (d >= today || done.has(d)) continue;
|
|
if (!byDay.has(d)) byDay.set(d, []);
|
|
byDay.get(d).push(r);
|
|
}
|
|
for (const [day, rows] of byDay) {
|
|
for (const [dim, key, n] of aggregateRows(rows)) insDaily.run(site, day, dim, key, n);
|
|
insDone.run(site, day);
|
|
stats.aggregated++;
|
|
}
|
|
// Fristen erst NACH der Verdichtung (Rohdaten und Klicks > 30 Tage)
|
|
db.prepare('DELETE FROM events_raw WHERE site = ? AND ts < ?').run(site, rawCut);
|
|
db.prepare('DELETE FROM clicks WHERE site = ? AND ts < ?').run(site, rawCut);
|
|
});
|
|
}
|
|
// Klicks ganz ohne Website-Zeile / Ereignisse
|
|
db.prepare('DELETE FROM clicks WHERE ts < ?').run(rawCut);
|
|
|
|
for (const { siteKey } of db.prepare('SELECT siteKey FROM sites WHERE offSince IS NOT NULL AND offSince < ?').all(offCut)) {
|
|
tx(() => {
|
|
for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) db.prepare(`DELETE FROM ${t} WHERE site = ?`).run(siteKey);
|
|
db.prepare('DELETE FROM sites WHERE siteKey = ?').run(siteKey);
|
|
stats.sitesDeleted++;
|
|
});
|
|
}
|
|
|
|
// Tageswerte älter als 25 Monate (alle Websites)
|
|
tx(() => {
|
|
const cut = dailyCutoffDay(now);
|
|
stats.dailyDeleted = Number(db.prepare('DELETE FROM daily WHERE day < ?').run(cut).changes);
|
|
db.prepare('DELETE FROM agg_state WHERE day < ?').run(cut);
|
|
});
|
|
|
|
// Ausgeschaltete Websites ohne offSince: Frist beginnt bei Entdeckung.
|
|
stats.orphansMarked = Number(db.prepare('UPDATE sites SET offSince = ? WHERE active = 0 AND offSince IS NULL').run(new Date(now).toISOString()).changes);
|
|
// Zeilen ohne Website-Eintrag
|
|
tx(() => {
|
|
for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) {
|
|
stats.orphanRowsDeleted += Number(db.prepare(`DELETE FROM ${t} WHERE site NOT IN (SELECT siteKey FROM sites)`).run().changes);
|
|
}
|
|
});
|
|
|
|
try { db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); } catch (e) { log.error('Analytics-Nachtjob: wal_checkpoint:', e.message); }
|
|
return stats;
|
|
}
|
|
|
|
// Offset (ms) von Europe/Berlin gegenüber UTC zum Zeitpunkt ts.
|
|
function berlinOffset(ts) {
|
|
const p = Object.fromEntries(WALL_FMT.formatToParts(ts).map((x) => [x.type, x.value]));
|
|
return Date.UTC(p.year, p.month - 1, p.day, p.hour, p.minute, p.second) - Math.floor(ts / 1000) * 1000;
|
|
}
|
|
|
|
// Nächster Lauf um hour:minute Europe/Berlin (strikt nach now), als UTC-ms.
|
|
export function nextRunAt(now, hour = 2, minute = 30) {
|
|
const [y, m, d] = dayOf(now).split('-').map(Number);
|
|
const at = (dd) => {
|
|
const guess = Date.UTC(y, m - 1, dd, hour, minute);
|
|
return guess - berlinOffset(guess - berlinOffset(guess));
|
|
};
|
|
const t = at(d);
|
|
return t > now ? t : at(d + 1);
|
|
}
|
|
|
|
// Plant runNightly täglich 02:30 Berlin und einmal kurz nach dem Start; stop() beendet alle Timer.
|
|
// Test-Hooks: now, timers ({ setTimeout, clearTimeout }) und job (statt runNightly).
|
|
export function scheduleNightly(db, { startDelayMs = 60000, log = console, now = Date.now, timers = { setTimeout, clearTimeout }, job = runNightly } = {}) {
|
|
let timer = null;
|
|
let stopped = false;
|
|
const run = () => {
|
|
try {
|
|
const r = job(db, { now: now() });
|
|
log.log(`Analytics-Nachtjob: ${r.aggregated} Tage verdichtet, ${r.sitesDeleted} Websites gelöscht, ${r.dailyDeleted} Tageswerte > ${DAILY_MONTHS} Monate gelöscht, ${r.orphansMarked} Websites markiert, ${r.orphanRowsDeleted} verwaiste Zeilen gelöscht`);
|
|
} catch (e) { log.error('Analytics-Nachtjob:', e.message); }
|
|
};
|
|
const arm = (delay, fn) => { if (!stopped) timer = timers.setTimeout(fn, delay); };
|
|
// Ein Timer kann früh feuern (Uhrsprung, Timer-Ungenauigkeit): dann nur neu auf den Zielzeitpunkt setzen,
|
|
// nie vorzeitig laufen. Nach dem Lauf strikt hinter das Ziel planen, damit 02:30 nie doppelt läuft.
|
|
const planDaily = (target) => arm(Math.max(0, target - now()), () => {
|
|
if (now() < target) return planDaily(target);
|
|
run();
|
|
planDaily(nextRunAt(Math.max(now(), target) + 1000));
|
|
});
|
|
arm(startDelayMs, () => { run(); planDaily(nextRunAt(now())); });
|
|
return { stop: () => { stopped = true; timers.clearTimeout(timer); } };
|
|
}
|