Analyse: Tageswerte nach 25 Monaten löschen, secure_delete, CSV-Export

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

analytics/scripts/export-site.mjs

@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+// CSV-Export einer Website (Tageswerte, Einzelereignisse, Klicks). Öffnet die Datenbank nur lesend.
+import { DatabaseSync } from 'node:sqlite';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const RE_KEY = /^[a-z0-9]{16,40}$/;
+const iso = (ts) => new Date(ts).toISOString();
+
+function cell(v) {
+  if (v === null || v === undefined) return '';
+  let s = String(v);
+  if (/^[=+\-@]/.test(s)) s = `'${s}`;
+  return /[",\r\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
+}
+const csv = (head, rows) => [head, ...rows].map((r) => r.map(cell).join(',')).join('\r\n') + '\r\n';
+
+export function exportSite({ dbPath, siteKey, outDir }) {
+  if (!RE_KEY.test(siteKey || '')) throw new Error('Ungültiger Website-Schlüssel');
+  const db = new DatabaseSync(dbPath, { readOnly: true });
+  try {
+    if (!db.prepare('SELECT 1 FROM sites WHERE siteKey = ?').get(siteKey)) throw new Error('Unbekannte Website');
+    const daily = db.prepare('SELECT day, dim, key, count FROM daily WHERE site = ? ORDER BY day, dim, key').all(siteKey);
+    const ev = db.prepare('SELECT ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind, host FROM events_raw WHERE site = ? ORDER BY ts, id').all(siteKey);
+    const cl = db.prepare('SELECT ts, path, vp, x, y, sel FROM clicks WHERE site = ? ORDER BY ts, id').all(siteKey);
+    mkdirSync(outDir, { recursive: true });
+    const files = {
+      [`${siteKey}-tageswerte.csv`]: csv(['day', 'dim', 'key', 'count'], daily.map((r) => [r.day, r.dim, r.key, r.count])),
+      [`${siteKey}-ereignisse.csv`]: csv(['time', 'type', 'path', 'vp', 'ref', 'country', 'device', 'visitor', 'sec', 'scroll', 'kind', 'host'],
+        ev.map((r) => [iso(r.ts), r.type, r.path, r.vp, r.ref, r.country, r.device, r.visitor, r.sec, r.scroll, r.kind, r.host])),
+      [`${siteKey}-klicks.csv`]: csv(['time', 'path', 'vp', 'x', 'y', 'sel'], cl.map((r) => [iso(r.ts), r.path, r.vp, r.x, r.y, r.sel])),
+    };
+    for (const [n, c] of Object.entries(files)) writeFileSync(join(outDir, n), c, 'utf8');
+    return Object.keys(files).map((n) => join(outDir, n));
+  } finally { db.close(); }
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  const args = process.argv.slice(2);
+  const opt = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; };
+  const siteKey = args.find((a, i) => !a.startsWith('--') && !(i > 0 && args[i - 1].startsWith('--')));
+  try {
+    const files = exportSite({ dbPath: opt('--db', '/srv/webklar-analytics/analytics.db'), siteKey, outDir: opt('--out', '.') });
+    console.log(files.join('\n'));
+  } catch (e) {
+    console.error(`Export fehlgeschlagen: ${e.message}`);
+    process.exit(1);
+  }
+}

analytics/src/db.js

@@ -55,6 +55,7 @@ CREATE TABLE IF NOT EXISTS agg_state (
 export function openDb(path) {
   const db = new DatabaseSync(path);
   if (path !== ':memory:') db.exec('PRAGMA journal_mode = WAL');
+  db.exec('PRAGMA secure_delete = ON');
   db.exec(SCHEMA);
   return db;
 }

analytics/src/jobs.js

@@ -8,6 +8,17 @@ const WALL_FMT = new Intl.DateTimeFormat('en-GB', {
 const DAY_MS = 86400 * 1000;
 export const RAW_DAYS = 30;
 export const OFF_DAYS = 60;
+export const DAILY_MONTHS = 25;
+
+// Berliner Tag von `now` minus DAILY_MONTHS Kalendermonate (Monatsende auf den letzten Tag des Zielmonats begrenzt).
+export function dailyCutoffDay(now) {
+  const [y, m, d] = DAY_FMT.format(now).split('-').map(Number);
+  const idx = y * 12 + (m - 1) - DAILY_MONTHS;
+  const ty = Math.floor(idx / 12);
+  const tm = idx % 12;
+  const last = new Date(Date.UTC(ty, tm + 1, 0)).getUTCDate();
+  return `${String(ty).padStart(4, '0')}-${String(tm + 1).padStart(2, '0')}-${String(Math.min(d, last)).padStart(2, '0')}`;
+}
 
 // Kalendertag (YYYY-MM-DD) in Europe/Berlin.
 export const dayOf = (ts) => DAY_FMT.format(ts);
@@ -38,7 +49,7 @@ export function aggregateRows(rows) {
   return [...m].filter(([, n]) => n > 0).map(([k, n]) => { const [dim, key] = k.split('\u0000'); return [dim, key, n]; });
 }
 
-export function runNightly(db, { now = Date.now() } = {}) {
+export function runNightly(db, { now = Date.now(), log = console } = {}) {
   const today = dayOf(now);
   const rawCut = now - RAW_DAYS * DAY_MS;
   const offCut = new Date(now - OFF_DAYS * DAY_MS).toISOString();
@@ -48,7 +59,7 @@ export function runNightly(db, { now = Date.now() } = {}) {
   const insDaily = db.prepare(`INSERT INTO daily (site, day, dim, key, count) VALUES (?, ?, ?, ?, ?)
     ON CONFLICT(site, day, dim, key) DO UPDATE SET count = count + excluded.count`);
   const insDone = db.prepare('INSERT OR IGNORE INTO agg_state (site, day) VALUES (?, ?)');
-  const stats = { aggregated: 0, sitesDeleted: 0 };
+  const stats = { aggregated: 0, sitesDeleted: 0, dailyDeleted: 0, orphansMarked: 0, orphanRowsDeleted: 0 };
 
   const tx = (fn) => {
     db.exec('BEGIN');
@@ -85,6 +96,24 @@ export function runNightly(db, { now = Date.now() } = {}) {
       stats.sitesDeleted++;
     });
   }
+
+  // Tageswerte älter als 25 Monate (alle Websites)
+  tx(() => {
+    const cut = dailyCutoffDay(now);
+    stats.dailyDeleted = Number(db.prepare('DELETE FROM daily WHERE day < ?').run(cut).changes);
+    db.prepare('DELETE FROM agg_state WHERE day < ?').run(cut);
+  });
+
+  // Ausgeschaltete Websites ohne offSince: Frist beginnt bei Entdeckung.
+  stats.orphansMarked = Number(db.prepare('UPDATE sites SET offSince = ? WHERE active = 0 AND offSince IS NULL').run(new Date(now).toISOString()).changes);
+  // Zeilen ohne Website-Eintrag
+  tx(() => {
+    for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) {
+      stats.orphanRowsDeleted += Number(db.prepare(`DELETE FROM ${t} WHERE site NOT IN (SELECT siteKey FROM sites)`).run().changes);
+    }
+  });
+
+  try { db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); } catch (e) { log.error('Analytics-Nachtjob: wal_checkpoint:', e.message); }
   return stats;
 }
 
@@ -113,7 +142,7 @@ export function scheduleNightly(db, { startDelayMs = 60000, log = console, now =
   const run = () => {
     try {
       const r = job(db, { now: now() });
-      log.log(`Analytics-Nachtjob: ${r.aggregated} Tage verdichtet, ${r.sitesDeleted} Websites gelöscht`);
+      log.log(`Analytics-Nachtjob: ${r.aggregated} Tage verdichtet, ${r.sitesDeleted} Websites gelöscht, ${r.dailyDeleted} Tageswerte > ${DAILY_MONTHS} Monate gelöscht, ${r.orphansMarked} Websites markiert, ${r.orphanRowsDeleted} verwaiste Zeilen gelöscht`);
     } catch (e) { log.error('Analytics-Nachtjob:', e.message); }
   };
   const arm = (delay, fn) => { if (!stopped) timer = timers.setTimeout(fn, delay); };

analytics/test/export.test.js

@@ -0,0 +1,73 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import crypto from 'node:crypto';
+import { mkdtempSync, readFileSync, readdirSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { openDb } from '../src/db.js';
+import { exportSite } from '../scripts/export-site.mjs';
+
+const KEY = 'abcdef0123456789abcd';
+const OTHER = 'zzzzzzzz0123456789zz';
+const SCRIPT = new URL('../scripts/export-site.mjs', import.meta.url).pathname;
+
+function setup() {
+  const dir = mkdtempSync(join(tmpdir(), 'exp-'));
+  const dbPath = join(dir, 'a.db');
+  const db = openDb(dbPath);
+  for (const k of [KEY, OTHER]) db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(k, 'p', '[]');
+  const ts = Date.UTC(2026, 8, 1, 10);
+  db.prepare("INSERT INTO daily VALUES (?, '2026-08-31','pv_total','',5)").run(KEY);
+  db.prepare("INSERT INTO daily VALUES (?, '2026-08-31','pv_path','/a',2)").run(KEY);
+  db.prepare("INSERT INTO daily VALUES (?, '2026-08-31','pv_total','',9)").run(OTHER);
+  db.prepare("INSERT INTO events_raw (site, ts, type, path, ref) VALUES (?,?, 'pv', '/x', ?)").run(KEY, ts, 'a,"b"\nc');
+  db.prepare("INSERT INTO events_raw (site, ts, type, path) VALUES (?,?, 'pv', '/o')").run(OTHER, ts);
+  db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y, sel) VALUES (?,?,?,?,?,?,?)').run(KEY, ts, '/', 'd', 1.5, 2, '=1+1');
+  db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y, sel) VALUES (?,?,?,?,?,?,?)').run(OTHER, ts, '/', 'd', 1, 2, 'x');
+  db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+  db.close();
+  return { dir, dbPath };
+}
+const hash = (p) => crypto.createHash('sha256').update(readFileSync(p)).digest('hex');
+
+test('Export: drei CSV-Dateien mit Kopfzeilen, nur diese Website, Quotes und Formelschutz', () => {
+  const { dir, dbPath } = setup();
+  const before = hash(dbPath);
+  const out = join(dir, 'out');
+  exportSite({ dbPath, siteKey: KEY, outDir: out });
+  assert.deepEqual(readdirSync(out).sort(), [`${KEY}-ereignisse.csv`, `${KEY}-klicks.csv`, `${KEY}-tageswerte.csv`]);
+  const t = readFileSync(join(out, `${KEY}-tageswerte.csv`), 'utf8');
+  assert.equal(t.split('\r\n')[0], 'day,dim,key,count');
+  assert.equal(t.split('\r\n').filter(Boolean).length, 3);
+  const e = readFileSync(join(out, `${KEY}-ereignisse.csv`), 'utf8');
+  assert.equal(e.split('\r\n')[0], 'time,type,path,vp,ref,country,device,visitor,sec,scroll,kind,host');
+  assert.ok(e.includes('2026-09-01T10:00:00.000Z,pv,/x,,"a,""b""\nc"'));
+  assert.ok(!e.includes('/o'));
+  const c = readFileSync(join(out, `${KEY}-klicks.csv`), 'utf8');
+  assert.equal(c.split('\r\n')[0], 'time,path,vp,x,y,sel');
+  assert.equal(c.split('\r\n').filter(Boolean).length, 2);
+  assert.ok(c.includes(",'=1+1"));
+  assert.equal(hash(dbPath), before);
+});
+
+test('Export: Datenbank ist nur lesend geöffnet', () => {
+  const { dbPath } = setup();
+  const ro = new DatabaseSync(dbPath, { readOnly: true });
+  assert.throws(() => ro.exec("INSERT INTO sites (siteKey, projectId, domains) VALUES ('q','p','[]')"));
+  ro.close();
+});
+
+test('CLI: unbekannter oder ungültiger Schlüssel => Exit 1 ohne Dateien', () => {
+  const { dir, dbPath } = setup();
+  for (const k of ['unbekannt0123456789ab', 'ABC']) {
+    const out = join(dir, 'o-' + k);
+    const r = spawnSync(process.execPath, [SCRIPT, k, '--db', dbPath, '--out', out]);
+    assert.equal(r.status, 1);
+    assert.throws(() => readdirSync(out));
+  }
+  const ok = spawnSync(process.execPath, [SCRIPT, KEY, '--db', dbPath, '--out', join(dir, 'ok')]);
+  assert.equal(ok.status, 0);
+  assert.equal(readdirSync(join(dir, 'ok')).length, 3);
+});

analytics/test/retention.test.js

@@ -0,0 +1,92 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, readFileSync, statSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { openDb } from '../src/db.js';
+import { runNightly, dailyCutoffDay, DAILY_MONTHS } from '../src/jobs.js';
+
+const KEY = 'abcdef0123456789abcd';
+const NOW = Date.UTC(2026, 9, 1, 10);
+const D = 86400 * 1000;
+
+test('dailyCutoffDay: 25 Kalendermonate zurück, Monatsende begrenzt', () => {
+  assert.equal(DAILY_MONTHS, 25);
+  assert.equal(dailyCutoffDay(Date.UTC(2026, 9, 1, 10)), '2024-09-01');
+  assert.equal(dailyCutoffDay(Date.UTC(2026, 2, 31, 10)), '2024-02-29');
+});
+
+function mk() {
+  const db = openDb(':memory:');
+  db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(KEY, 'p', '["a.de"]');
+  return db;
+}
+
+test('Tageswerte älter als 25 Monate werden gelöscht, Grenztag bleibt', () => {
+  const db = mk();
+  for (const day of ['2024-08-31', '2024-09-01']) {
+    db.prepare("INSERT INTO daily (site, day, dim, key, count) VALUES (?,?, 'pv_total','',3)").run(KEY, day);
+    db.prepare('INSERT INTO agg_state (site, day) VALUES (?,?)').run(KEY, day);
+  }
+  const r = runNightly(db, { now: NOW });
+  assert.deepEqual(db.prepare('SELECT day FROM daily').all().map((x) => x.day), ['2024-09-01']);
+  assert.deepEqual(db.prepare('SELECT day FROM agg_state').all().map((x) => x.day), ['2024-09-01']);
+  assert.equal(r.dailyDeleted, 1);
+});
+
+test('ausgeschaltete Website ohne offSince bekommt offSince, zweiter Lauf ändert nichts', () => {
+  const db = mk();
+  db.prepare('UPDATE sites SET active = 0').run();
+  const r = runNightly(db, { now: NOW });
+  assert.equal(db.prepare('SELECT offSince FROM sites').get().offSince, new Date(NOW).toISOString());
+  assert.equal(r.orphansMarked, 1);
+  runNightly(db, { now: NOW + D });
+  assert.equal(db.prepare('SELECT offSince FROM sites').get().offSince, new Date(NOW).toISOString());
+});
+
+test('Website mit offSince > 60 Tage wird weiter komplett gelöscht', () => {
+  const db = mk();
+  db.prepare('UPDATE sites SET active = 0, offSince = ?').run(new Date(NOW - 61 * D).toISOString());
+  const r = runNightly(db, { now: NOW });
+  assert.equal(r.sitesDeleted, 1);
+  assert.equal(db.prepare('SELECT COUNT(*) n FROM sites').get().n, 0);
+});
+
+test('Zeilen ohne sites-Eintrag werden gelöscht, andere bleiben', () => {
+  const db = mk();
+  const GONE = 'gonegonegonegonegone';
+  for (const s of [KEY, GONE]) {
+    db.prepare("INSERT INTO events_raw (site, ts, type) VALUES (?,?, 'pv')").run(s, NOW - 1000);
+    db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,?,?)').run(s, NOW - 1000, '/', 'd', 1, 1);
+    db.prepare("INSERT INTO daily (site, day, dim, key, count) VALUES (?, '2026-09-01','pv_total','',1)").run(s);
+    db.prepare("INSERT INTO agg_state (site, day) VALUES (?, '2026-09-01')").run(s);
+  }
+  const r = runNightly(db, { now: NOW });
+  for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) {
+    assert.deepEqual(db.prepare(`SELECT DISTINCT site FROM ${t}`).all().map((x) => x.site), [KEY], t);
+  }
+  assert.ok(r.orphanRowsDeleted >= 4);
+});
+
+test('secure_delete: gelöschter Pfad steht nicht mehr in der Datei', () => {
+  const dir = mkdtempSync(join(tmpdir(), 'sd-'));
+  const p = join(dir, 'a.db');
+  const db = openDb(p);
+  assert.equal(db.prepare('PRAGMA secure_delete').get().secure_delete, 1);
+  db.prepare("INSERT INTO events_raw (site, ts, type, path) VALUES ('x', 1, 'pv', 'GEHEIMER-PFAD-4711')").run();
+  db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+  db.exec("DELETE FROM events_raw WHERE path = 'GEHEIMER-PFAD-4711'");
+  db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+  assert.ok(!readFileSync(p).includes('GEHEIMER-PFAD-4711'));
+  db.close();
+});
+
+test('runNightly leert das WAL', () => {
+  const dir = mkdtempSync(join(tmpdir(), 'wal-'));
+  const p = join(dir, 'a.db');
+  const db = openDb(p);
+  db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(KEY, 'p', '[]');
+  runNightly(db, { now: NOW });
+  assert.equal(statSync(p + '-wal').size, 0);
+  db.close();
+});