diff --git a/analytics/scripts/export-site.mjs b/analytics/scripts/export-site.mjs new file mode 100644 index 0000000..c2f7ad4 --- /dev/null +++ b/analytics/scripts/export-site.mjs @@ -0,0 +1,50 @@ +#!/usr/bin/env node +// CSV-Export einer Website (Tageswerte, Einzelereignisse, Klicks). Öffnet die Datenbank nur lesend. +import { DatabaseSync } from 'node:sqlite'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const RE_KEY = /^[a-z0-9]{16,40}$/; +const iso = (ts) => new Date(ts).toISOString(); + +function cell(v) { + if (v === null || v === undefined) return ''; + let s = String(v); + if (/^[=+\-@]/.test(s)) s = `'${s}`; + return /[",\r\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; +} +const csv = (head, rows) => [head, ...rows].map((r) => r.map(cell).join(',')).join('\r\n') + '\r\n'; + +export function exportSite({ dbPath, siteKey, outDir }) { + if (!RE_KEY.test(siteKey || '')) throw new Error('Ungültiger Website-Schlüssel'); + const db = new DatabaseSync(dbPath, { readOnly: true }); + try { + if (!db.prepare('SELECT 1 FROM sites WHERE siteKey = ?').get(siteKey)) throw new Error('Unbekannte Website'); + const daily = db.prepare('SELECT day, dim, key, count FROM daily WHERE site = ? ORDER BY day, dim, key').all(siteKey); + const ev = db.prepare('SELECT ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind, host FROM events_raw WHERE site = ? ORDER BY ts, id').all(siteKey); + const cl = db.prepare('SELECT ts, path, vp, x, y, sel FROM clicks WHERE site = ? ORDER BY ts, id').all(siteKey); + mkdirSync(outDir, { recursive: true }); + const files = { + [`${siteKey}-tageswerte.csv`]: csv(['day', 'dim', 'key', 'count'], daily.map((r) => [r.day, r.dim, r.key, r.count])), + [`${siteKey}-ereignisse.csv`]: csv(['time', 'type', 'path', 'vp', 'ref', 'country', 'device', 'visitor', 'sec', 'scroll', 'kind', 'host'], + ev.map((r) => [iso(r.ts), r.type, r.path, r.vp, r.ref, r.country, r.device, r.visitor, r.sec, r.scroll, r.kind, r.host])), + [`${siteKey}-klicks.csv`]: csv(['time', 'path', 'vp', 'x', 'y', 'sel'], cl.map((r) => [iso(r.ts), r.path, r.vp, r.x, r.y, r.sel])), + }; + for (const [n, c] of Object.entries(files)) writeFileSync(join(outDir, n), c, 'utf8'); + return Object.keys(files).map((n) => join(outDir, n)); + } finally { db.close(); } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const args = process.argv.slice(2); + const opt = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; }; + const siteKey = args.find((a, i) => !a.startsWith('--') && !(i > 0 && args[i - 1].startsWith('--'))); + try { + const files = exportSite({ dbPath: opt('--db', '/srv/webklar-analytics/analytics.db'), siteKey, outDir: opt('--out', '.') }); + console.log(files.join('\n')); + } catch (e) { + console.error(`Export fehlgeschlagen: ${e.message}`); + process.exit(1); + } +} diff --git a/analytics/src/db.js b/analytics/src/db.js index a43ace4..9f67ae6 100644 --- a/analytics/src/db.js +++ b/analytics/src/db.js @@ -55,6 +55,7 @@ CREATE TABLE IF NOT EXISTS agg_state ( export function openDb(path) { const db = new DatabaseSync(path); if (path !== ':memory:') db.exec('PRAGMA journal_mode = WAL'); + db.exec('PRAGMA secure_delete = ON'); db.exec(SCHEMA); return db; } diff --git a/analytics/src/jobs.js b/analytics/src/jobs.js index ccb8fbb..8781d5f 100644 --- a/analytics/src/jobs.js +++ b/analytics/src/jobs.js @@ -8,6 +8,17 @@ const WALL_FMT = new Intl.DateTimeFormat('en-GB', { const DAY_MS = 86400 * 1000; export const RAW_DAYS = 30; export const OFF_DAYS = 60; +export const DAILY_MONTHS = 25; + +// Berliner Tag von `now` minus DAILY_MONTHS Kalendermonate (Monatsende auf den letzten Tag des Zielmonats begrenzt). +export function dailyCutoffDay(now) { + const [y, m, d] = DAY_FMT.format(now).split('-').map(Number); + const idx = y * 12 + (m - 1) - DAILY_MONTHS; + const ty = Math.floor(idx / 12); + const tm = idx % 12; + const last = new Date(Date.UTC(ty, tm + 1, 0)).getUTCDate(); + return `${String(ty).padStart(4, '0')}-${String(tm + 1).padStart(2, '0')}-${String(Math.min(d, last)).padStart(2, '0')}`; +} // Kalendertag (YYYY-MM-DD) in Europe/Berlin. export const dayOf = (ts) => DAY_FMT.format(ts); @@ -38,7 +49,7 @@ export function aggregateRows(rows) { return [...m].filter(([, n]) => n > 0).map(([k, n]) => { const [dim, key] = k.split('\u0000'); return [dim, key, n]; }); } -export function runNightly(db, { now = Date.now() } = {}) { +export function runNightly(db, { now = Date.now(), log = console } = {}) { const today = dayOf(now); const rawCut = now - RAW_DAYS * DAY_MS; const offCut = new Date(now - OFF_DAYS * DAY_MS).toISOString(); @@ -48,7 +59,7 @@ export function runNightly(db, { now = Date.now() } = {}) { const insDaily = db.prepare(`INSERT INTO daily (site, day, dim, key, count) VALUES (?, ?, ?, ?, ?) ON CONFLICT(site, day, dim, key) DO UPDATE SET count = count + excluded.count`); const insDone = db.prepare('INSERT OR IGNORE INTO agg_state (site, day) VALUES (?, ?)'); - const stats = { aggregated: 0, sitesDeleted: 0 }; + const stats = { aggregated: 0, sitesDeleted: 0, dailyDeleted: 0, orphansMarked: 0, orphanRowsDeleted: 0 }; const tx = (fn) => { db.exec('BEGIN'); @@ -85,6 +96,24 @@ export function runNightly(db, { now = Date.now() } = {}) { stats.sitesDeleted++; }); } + + // Tageswerte älter als 25 Monate (alle Websites) + tx(() => { + const cut = dailyCutoffDay(now); + stats.dailyDeleted = Number(db.prepare('DELETE FROM daily WHERE day < ?').run(cut).changes); + db.prepare('DELETE FROM agg_state WHERE day < ?').run(cut); + }); + + // Ausgeschaltete Websites ohne offSince: Frist beginnt bei Entdeckung. + stats.orphansMarked = Number(db.prepare('UPDATE sites SET offSince = ? WHERE active = 0 AND offSince IS NULL').run(new Date(now).toISOString()).changes); + // Zeilen ohne Website-Eintrag + tx(() => { + for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) { + stats.orphanRowsDeleted += Number(db.prepare(`DELETE FROM ${t} WHERE site NOT IN (SELECT siteKey FROM sites)`).run().changes); + } + }); + + try { db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); } catch (e) { log.error('Analytics-Nachtjob: wal_checkpoint:', e.message); } return stats; } @@ -113,7 +142,7 @@ export function scheduleNightly(db, { startDelayMs = 60000, log = console, now = const run = () => { try { const r = job(db, { now: now() }); - log.log(`Analytics-Nachtjob: ${r.aggregated} Tage verdichtet, ${r.sitesDeleted} Websites gelöscht`); + log.log(`Analytics-Nachtjob: ${r.aggregated} Tage verdichtet, ${r.sitesDeleted} Websites gelöscht, ${r.dailyDeleted} Tageswerte > ${DAILY_MONTHS} Monate gelöscht, ${r.orphansMarked} Websites markiert, ${r.orphanRowsDeleted} verwaiste Zeilen gelöscht`); } catch (e) { log.error('Analytics-Nachtjob:', e.message); } }; const arm = (delay, fn) => { if (!stopped) timer = timers.setTimeout(fn, delay); }; diff --git a/analytics/test/export.test.js b/analytics/test/export.test.js new file mode 100644 index 0000000..142290b --- /dev/null +++ b/analytics/test/export.test.js @@ -0,0 +1,73 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; +import { mkdtempSync, readFileSync, readdirSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { openDb } from '../src/db.js'; +import { exportSite } from '../scripts/export-site.mjs'; + +const KEY = 'abcdef0123456789abcd'; +const OTHER = 'zzzzzzzz0123456789zz'; +const SCRIPT = new URL('../scripts/export-site.mjs', import.meta.url).pathname; + +function setup() { + const dir = mkdtempSync(join(tmpdir(), 'exp-')); + const dbPath = join(dir, 'a.db'); + const db = openDb(dbPath); + for (const k of [KEY, OTHER]) db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(k, 'p', '[]'); + const ts = Date.UTC(2026, 8, 1, 10); + db.prepare("INSERT INTO daily VALUES (?, '2026-08-31','pv_total','',5)").run(KEY); + db.prepare("INSERT INTO daily VALUES (?, '2026-08-31','pv_path','/a',2)").run(KEY); + db.prepare("INSERT INTO daily VALUES (?, '2026-08-31','pv_total','',9)").run(OTHER); + db.prepare("INSERT INTO events_raw (site, ts, type, path, ref) VALUES (?,?, 'pv', '/x', ?)").run(KEY, ts, 'a,"b"\nc'); + db.prepare("INSERT INTO events_raw (site, ts, type, path) VALUES (?,?, 'pv', '/o')").run(OTHER, ts); + db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y, sel) VALUES (?,?,?,?,?,?,?)').run(KEY, ts, '/', 'd', 1.5, 2, '=1+1'); + db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y, sel) VALUES (?,?,?,?,?,?,?)').run(OTHER, ts, '/', 'd', 1, 2, 'x'); + db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); + db.close(); + return { dir, dbPath }; +} +const hash = (p) => crypto.createHash('sha256').update(readFileSync(p)).digest('hex'); + +test('Export: drei CSV-Dateien mit Kopfzeilen, nur diese Website, Quotes und Formelschutz', () => { + const { dir, dbPath } = setup(); + const before = hash(dbPath); + const out = join(dir, 'out'); + exportSite({ dbPath, siteKey: KEY, outDir: out }); + assert.deepEqual(readdirSync(out).sort(), [`${KEY}-ereignisse.csv`, `${KEY}-klicks.csv`, `${KEY}-tageswerte.csv`]); + const t = readFileSync(join(out, `${KEY}-tageswerte.csv`), 'utf8'); + assert.equal(t.split('\r\n')[0], 'day,dim,key,count'); + assert.equal(t.split('\r\n').filter(Boolean).length, 3); + const e = readFileSync(join(out, `${KEY}-ereignisse.csv`), 'utf8'); + assert.equal(e.split('\r\n')[0], 'time,type,path,vp,ref,country,device,visitor,sec,scroll,kind,host'); + assert.ok(e.includes('2026-09-01T10:00:00.000Z,pv,/x,,"a,""b""\nc"')); + assert.ok(!e.includes('/o')); + const c = readFileSync(join(out, `${KEY}-klicks.csv`), 'utf8'); + assert.equal(c.split('\r\n')[0], 'time,path,vp,x,y,sel'); + assert.equal(c.split('\r\n').filter(Boolean).length, 2); + assert.ok(c.includes(",'=1+1")); + assert.equal(hash(dbPath), before); +}); + +test('Export: Datenbank ist nur lesend geöffnet', () => { + const { dbPath } = setup(); + const ro = new DatabaseSync(dbPath, { readOnly: true }); + assert.throws(() => ro.exec("INSERT INTO sites (siteKey, projectId, domains) VALUES ('q','p','[]')")); + ro.close(); +}); + +test('CLI: unbekannter oder ungültiger Schlüssel => Exit 1 ohne Dateien', () => { + const { dir, dbPath } = setup(); + for (const k of ['unbekannt0123456789ab', 'ABC']) { + const out = join(dir, 'o-' + k); + const r = spawnSync(process.execPath, [SCRIPT, k, '--db', dbPath, '--out', out]); + assert.equal(r.status, 1); + assert.throws(() => readdirSync(out)); + } + const ok = spawnSync(process.execPath, [SCRIPT, KEY, '--db', dbPath, '--out', join(dir, 'ok')]); + assert.equal(ok.status, 0); + assert.equal(readdirSync(join(dir, 'ok')).length, 3); +}); diff --git a/analytics/test/retention.test.js b/analytics/test/retention.test.js new file mode 100644 index 0000000..567b101 --- /dev/null +++ b/analytics/test/retention.test.js @@ -0,0 +1,92 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { openDb } from '../src/db.js'; +import { runNightly, dailyCutoffDay, DAILY_MONTHS } from '../src/jobs.js'; + +const KEY = 'abcdef0123456789abcd'; +const NOW = Date.UTC(2026, 9, 1, 10); +const D = 86400 * 1000; + +test('dailyCutoffDay: 25 Kalendermonate zurück, Monatsende begrenzt', () => { + assert.equal(DAILY_MONTHS, 25); + assert.equal(dailyCutoffDay(Date.UTC(2026, 9, 1, 10)), '2024-09-01'); + assert.equal(dailyCutoffDay(Date.UTC(2026, 2, 31, 10)), '2024-02-29'); +}); + +function mk() { + const db = openDb(':memory:'); + db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(KEY, 'p', '["a.de"]'); + return db; +} + +test('Tageswerte älter als 25 Monate werden gelöscht, Grenztag bleibt', () => { + const db = mk(); + for (const day of ['2024-08-31', '2024-09-01']) { + db.prepare("INSERT INTO daily (site, day, dim, key, count) VALUES (?,?, 'pv_total','',3)").run(KEY, day); + db.prepare('INSERT INTO agg_state (site, day) VALUES (?,?)').run(KEY, day); + } + const r = runNightly(db, { now: NOW }); + assert.deepEqual(db.prepare('SELECT day FROM daily').all().map((x) => x.day), ['2024-09-01']); + assert.deepEqual(db.prepare('SELECT day FROM agg_state').all().map((x) => x.day), ['2024-09-01']); + assert.equal(r.dailyDeleted, 1); +}); + +test('ausgeschaltete Website ohne offSince bekommt offSince, zweiter Lauf ändert nichts', () => { + const db = mk(); + db.prepare('UPDATE sites SET active = 0').run(); + const r = runNightly(db, { now: NOW }); + assert.equal(db.prepare('SELECT offSince FROM sites').get().offSince, new Date(NOW).toISOString()); + assert.equal(r.orphansMarked, 1); + runNightly(db, { now: NOW + D }); + assert.equal(db.prepare('SELECT offSince FROM sites').get().offSince, new Date(NOW).toISOString()); +}); + +test('Website mit offSince > 60 Tage wird weiter komplett gelöscht', () => { + const db = mk(); + db.prepare('UPDATE sites SET active = 0, offSince = ?').run(new Date(NOW - 61 * D).toISOString()); + const r = runNightly(db, { now: NOW }); + assert.equal(r.sitesDeleted, 1); + assert.equal(db.prepare('SELECT COUNT(*) n FROM sites').get().n, 0); +}); + +test('Zeilen ohne sites-Eintrag werden gelöscht, andere bleiben', () => { + const db = mk(); + const GONE = 'gonegonegonegonegone'; + for (const s of [KEY, GONE]) { + db.prepare("INSERT INTO events_raw (site, ts, type) VALUES (?,?, 'pv')").run(s, NOW - 1000); + db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,?,?)').run(s, NOW - 1000, '/', 'd', 1, 1); + db.prepare("INSERT INTO daily (site, day, dim, key, count) VALUES (?, '2026-09-01','pv_total','',1)").run(s); + db.prepare("INSERT INTO agg_state (site, day) VALUES (?, '2026-09-01')").run(s); + } + const r = runNightly(db, { now: NOW }); + for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) { + assert.deepEqual(db.prepare(`SELECT DISTINCT site FROM ${t}`).all().map((x) => x.site), [KEY], t); + } + assert.ok(r.orphanRowsDeleted >= 4); +}); + +test('secure_delete: gelöschter Pfad steht nicht mehr in der Datei', () => { + const dir = mkdtempSync(join(tmpdir(), 'sd-')); + const p = join(dir, 'a.db'); + const db = openDb(p); + assert.equal(db.prepare('PRAGMA secure_delete').get().secure_delete, 1); + db.prepare("INSERT INTO events_raw (site, ts, type, path) VALUES ('x', 1, 'pv', 'GEHEIMER-PFAD-4711')").run(); + db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); + db.exec("DELETE FROM events_raw WHERE path = 'GEHEIMER-PFAD-4711'"); + db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); + assert.ok(!readFileSync(p).includes('GEHEIMER-PFAD-4711')); + db.close(); +}); + +test('runNightly leert das WAL', () => { + const dir = mkdtempSync(join(tmpdir(), 'wal-')); + const p = join(dir, 'a.db'); + const db = openDb(p); + db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(KEY, 'p', '[]'); + runNightly(db, { now: NOW }); + assert.equal(statSync(p + '-wal').size, 0); + db.close(); +});