Analyse: Tageswerte nach 25 Monaten löschen, secure_delete, CSV-Export
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ClaudeandClaude Opus 5.5 committed
1 parent
a73e0be952
commit
97890a8eee
5 files changed
+248
-3
No files matched your search
@@ -0,0 +1,73 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import crypto from 'node:crypto';
|
||||
import { mkdtempSync, readFileSync, readdirSync } from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { openDb } from '../src/db.js';
|
||||
import { exportSite } from '../scripts/export-site.mjs';
|
||||
|
||||
const KEY = 'abcdef0123456789abcd';
|
||||
const OTHER = 'zzzzzzzz0123456789zz';
|
||||
const SCRIPT = new URL('../scripts/export-site.mjs', import.meta.url).pathname;
|
||||
|
||||
function setup() {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'exp-'));
|
||||
const dbPath = join(dir, 'a.db');
|
||||
const db = openDb(dbPath);
|
||||
for (const k of [KEY, OTHER]) db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(k, 'p', '[]');
|
||||
const ts = Date.UTC(2026, 8, 1, 10);
|
||||
db.prepare("INSERT INTO daily VALUES (?, '2026-08-31','pv_total','',5)").run(KEY);
|
||||
db.prepare("INSERT INTO daily VALUES (?, '2026-08-31','pv_path','/a',2)").run(KEY);
|
||||
db.prepare("INSERT INTO daily VALUES (?, '2026-08-31','pv_total','',9)").run(OTHER);
|
||||
db.prepare("INSERT INTO events_raw (site, ts, type, path, ref) VALUES (?,?, 'pv', '/x', ?)").run(KEY, ts, 'a,"b"\nc');
|
||||
db.prepare("INSERT INTO events_raw (site, ts, type, path) VALUES (?,?, 'pv', '/o')").run(OTHER, ts);
|
||||
db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y, sel) VALUES (?,?,?,?,?,?,?)').run(KEY, ts, '/', 'd', 1.5, 2, '=1+1');
|
||||
db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y, sel) VALUES (?,?,?,?,?,?,?)').run(OTHER, ts, '/', 'd', 1, 2, 'x');
|
||||
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
db.close();
|
||||
return { dir, dbPath };
|
||||
}
|
||||
const hash = (p) => crypto.createHash('sha256').update(readFileSync(p)).digest('hex');
|
||||
|
||||
test('Export: drei CSV-Dateien mit Kopfzeilen, nur diese Website, Quotes und Formelschutz', () => {
|
||||
const { dir, dbPath } = setup();
|
||||
const before = hash(dbPath);
|
||||
const out = join(dir, 'out');
|
||||
exportSite({ dbPath, siteKey: KEY, outDir: out });
|
||||
assert.deepEqual(readdirSync(out).sort(), [`${KEY}-ereignisse.csv`, `${KEY}-klicks.csv`, `${KEY}-tageswerte.csv`]);
|
||||
const t = readFileSync(join(out, `${KEY}-tageswerte.csv`), 'utf8');
|
||||
assert.equal(t.split('\r\n')[0], 'day,dim,key,count');
|
||||
assert.equal(t.split('\r\n').filter(Boolean).length, 3);
|
||||
const e = readFileSync(join(out, `${KEY}-ereignisse.csv`), 'utf8');
|
||||
assert.equal(e.split('\r\n')[0], 'time,type,path,vp,ref,country,device,visitor,sec,scroll,kind,host');
|
||||
assert.ok(e.includes('2026-09-01T10:00:00.000Z,pv,/x,,"a,""b""\nc"'));
|
||||
assert.ok(!e.includes('/o'));
|
||||
const c = readFileSync(join(out, `${KEY}-klicks.csv`), 'utf8');
|
||||
assert.equal(c.split('\r\n')[0], 'time,path,vp,x,y,sel');
|
||||
assert.equal(c.split('\r\n').filter(Boolean).length, 2);
|
||||
assert.ok(c.includes(",'=1+1"));
|
||||
assert.equal(hash(dbPath), before);
|
||||
});
|
||||
|
||||
test('Export: Datenbank ist nur lesend geöffnet', () => {
|
||||
const { dbPath } = setup();
|
||||
const ro = new DatabaseSync(dbPath, { readOnly: true });
|
||||
assert.throws(() => ro.exec("INSERT INTO sites (siteKey, projectId, domains) VALUES ('q','p','[]')"));
|
||||
ro.close();
|
||||
});
|
||||
|
||||
test('CLI: unbekannter oder ungültiger Schlüssel => Exit 1 ohne Dateien', () => {
|
||||
const { dir, dbPath } = setup();
|
||||
for (const k of ['unbekannt0123456789ab', 'ABC']) {
|
||||
const out = join(dir, 'o-' + k);
|
||||
const r = spawnSync(process.execPath, [SCRIPT, k, '--db', dbPath, '--out', out]);
|
||||
assert.equal(r.status, 1);
|
||||
assert.throws(() => readdirSync(out));
|
||||
}
|
||||
const ok = spawnSync(process.execPath, [SCRIPT, KEY, '--db', dbPath, '--out', join(dir, 'ok')]);
|
||||
assert.equal(ok.status, 0);
|
||||
assert.equal(readdirSync(join(dir, 'ok')).length, 3);
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, readFileSync, statSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { openDb } from '../src/db.js';
|
||||
import { runNightly, dailyCutoffDay, DAILY_MONTHS } from '../src/jobs.js';
|
||||
|
||||
const KEY = 'abcdef0123456789abcd';
|
||||
const NOW = Date.UTC(2026, 9, 1, 10);
|
||||
const D = 86400 * 1000;
|
||||
|
||||
test('dailyCutoffDay: 25 Kalendermonate zurück, Monatsende begrenzt', () => {
|
||||
assert.equal(DAILY_MONTHS, 25);
|
||||
assert.equal(dailyCutoffDay(Date.UTC(2026, 9, 1, 10)), '2024-09-01');
|
||||
assert.equal(dailyCutoffDay(Date.UTC(2026, 2, 31, 10)), '2024-02-29');
|
||||
});
|
||||
|
||||
function mk() {
|
||||
const db = openDb(':memory:');
|
||||
db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(KEY, 'p', '["a.de"]');
|
||||
return db;
|
||||
}
|
||||
|
||||
test('Tageswerte älter als 25 Monate werden gelöscht, Grenztag bleibt', () => {
|
||||
const db = mk();
|
||||
for (const day of ['2024-08-31', '2024-09-01']) {
|
||||
db.prepare("INSERT INTO daily (site, day, dim, key, count) VALUES (?,?, 'pv_total','',3)").run(KEY, day);
|
||||
db.prepare('INSERT INTO agg_state (site, day) VALUES (?,?)').run(KEY, day);
|
||||
}
|
||||
const r = runNightly(db, { now: NOW });
|
||||
assert.deepEqual(db.prepare('SELECT day FROM daily').all().map((x) => x.day), ['2024-09-01']);
|
||||
assert.deepEqual(db.prepare('SELECT day FROM agg_state').all().map((x) => x.day), ['2024-09-01']);
|
||||
assert.equal(r.dailyDeleted, 1);
|
||||
});
|
||||
|
||||
test('ausgeschaltete Website ohne offSince bekommt offSince, zweiter Lauf ändert nichts', () => {
|
||||
const db = mk();
|
||||
db.prepare('UPDATE sites SET active = 0').run();
|
||||
const r = runNightly(db, { now: NOW });
|
||||
assert.equal(db.prepare('SELECT offSince FROM sites').get().offSince, new Date(NOW).toISOString());
|
||||
assert.equal(r.orphansMarked, 1);
|
||||
runNightly(db, { now: NOW + D });
|
||||
assert.equal(db.prepare('SELECT offSince FROM sites').get().offSince, new Date(NOW).toISOString());
|
||||
});
|
||||
|
||||
test('Website mit offSince > 60 Tage wird weiter komplett gelöscht', () => {
|
||||
const db = mk();
|
||||
db.prepare('UPDATE sites SET active = 0, offSince = ?').run(new Date(NOW - 61 * D).toISOString());
|
||||
const r = runNightly(db, { now: NOW });
|
||||
assert.equal(r.sitesDeleted, 1);
|
||||
assert.equal(db.prepare('SELECT COUNT(*) n FROM sites').get().n, 0);
|
||||
});
|
||||
|
||||
test('Zeilen ohne sites-Eintrag werden gelöscht, andere bleiben', () => {
|
||||
const db = mk();
|
||||
const GONE = 'gonegonegonegonegone';
|
||||
for (const s of [KEY, GONE]) {
|
||||
db.prepare("INSERT INTO events_raw (site, ts, type) VALUES (?,?, 'pv')").run(s, NOW - 1000);
|
||||
db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,?,?)').run(s, NOW - 1000, '/', 'd', 1, 1);
|
||||
db.prepare("INSERT INTO daily (site, day, dim, key, count) VALUES (?, '2026-09-01','pv_total','',1)").run(s);
|
||||
db.prepare("INSERT INTO agg_state (site, day) VALUES (?, '2026-09-01')").run(s);
|
||||
}
|
||||
const r = runNightly(db, { now: NOW });
|
||||
for (const t of ['events_raw', 'clicks', 'daily', 'agg_state']) {
|
||||
assert.deepEqual(db.prepare(`SELECT DISTINCT site FROM ${t}`).all().map((x) => x.site), [KEY], t);
|
||||
}
|
||||
assert.ok(r.orphanRowsDeleted >= 4);
|
||||
});
|
||||
|
||||
test('secure_delete: gelöschter Pfad steht nicht mehr in der Datei', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'sd-'));
|
||||
const p = join(dir, 'a.db');
|
||||
const db = openDb(p);
|
||||
assert.equal(db.prepare('PRAGMA secure_delete').get().secure_delete, 1);
|
||||
db.prepare("INSERT INTO events_raw (site, ts, type, path) VALUES ('x', 1, 'pv', 'GEHEIMER-PFAD-4711')").run();
|
||||
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
db.exec("DELETE FROM events_raw WHERE path = 'GEHEIMER-PFAD-4711'");
|
||||
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
assert.ok(!readFileSync(p).includes('GEHEIMER-PFAD-4711'));
|
||||
db.close();
|
||||
});
|
||||
|
||||
test('runNightly leert das WAL', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'wal-'));
|
||||
const p = join(dir, 'a.db');
|
||||
const db = openDb(p);
|
||||
db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(KEY, 'p', '[]');
|
||||
runNightly(db, { now: NOW });
|
||||
assert.equal(statSync(p + '-wal').size, 0);
|
||||
db.close();
|
||||
});
|
||||
Reference in new issue
Block a user