Webklar-Kundenbereich/analytics/test/stats.test.js

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { openDb } from '../src/db.js';
import { runNightly, nextRunAt, dayOf, scheduleNightly } from '../src/jobs.js';
import { createAdminServer } from '../src/server.js';

const KEY = 'abcdef0123456789abcd';
const OTHER = 'zzzzzzzz0123456789zz';
const NOW = Date.UTC(2026, 9, 10, 10, 0, 0); // 10.10.2026 12:00 Berlin
const H = 3600 * 1000;
const D = 24 * H;

function mk() {
  const db = openDb(':memory:');
  db.prepare('INSERT INTO sites (siteKey, projectId, domains, active) VALUES (?,?,?,1)').run(KEY, 'p', '["a.de"]');
  const ev = (o) => db.prepare(`INSERT INTO events_raw (site, ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind, host)
    VALUES (@site,@ts,@type,@path,@vp,@ref,@country,@device,@visitor,@sec,@scroll,@kind,@host)`)
    .run({ site: KEY, path: null, vp: null, ref: null, country: null, device: null, visitor: null, sec: null, scroll: null, kind: null, host: null, ...o });
  return { db, ev };
}
const daily = (db, site = KEY) => db.prepare('SELECT day, dim, key, count FROM daily WHERE site = ? ORDER BY day, dim, key').all(site);
const get = (db, day, dim, key = '') => db.prepare('SELECT count FROM daily WHERE site=? AND day=? AND dim=? AND key=?').get(KEY, day, dim, key)?.count;

test('Verdichtung: Dimensionen, nur Tage < heute, idempotent', () => {
  const { db, ev } = mk();
  const t = NOW - D; // 9.10.
  ev({ ts: t, type: 'pv', path: '/', vp: 'd', ref: 'google.com', country: 'DE', device: 'd' });
  ev({ ts: t + 1, type: 'pv', path: '/', vp: 'm', ref: null, country: 'DE', device: 'm' });
  ev({ ts: t + 2, type: 'pv', path: '/x', vp: 'd', ref: 'google.com', country: 'AT', device: 'd' });
  ev({ ts: t + 3, type: 'en', path: '/', vp: 'd', visitor: 'v1', sec: 10, scroll: 50 });
  ev({ ts: t + 4, type: 'en', path: '/x', vp: 'd', visitor: 'v1', sec: 20, scroll: 70 });
  ev({ ts: t + 5, type: 'ct', path: '/', visitor: 'v2', kind: 'tel' });
  ev({ ts: t + 6, type: 'ct', path: '/', visitor: 'v2', kind: 'out', host: 'x.de' });
  ev({ ts: t + 7, type: 'bn', kind: 'shown' });
  ev({ ts: t + 8, type: 'bn', kind: 'accept' });
  ev({ ts: NOW, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); // heute: unberührt
  runNightly(db, { now: NOW });
  const d = '2026-10-09';
  assert.equal(get(db, d, 'pv_total'), 3);
  assert.equal(get(db, d, 'pv_path', '/'), 2);
  assert.equal(get(db, d, 'pv_path', '/x'), 1);
  assert.equal(get(db, d, 'pv_ref', 'google.com'), 2);
  assert.equal(get(db, d, 'pv_ref', ''), 1);
  assert.equal(get(db, d, 'pv_country', 'DE'), 2);
  assert.equal(get(db, d, 'pv_device', 'm'), 1);
  assert.equal(get(db, d, 'visitors'), 2); // v1, v2
  assert.equal(get(db, d, 'en_count'), 2);
  assert.equal(get(db, d, 'en_sec_sum'), 30);
  assert.equal(get(db, d, 'scroll_sum'), 120);
  assert.equal(get(db, d, 'ct_kind', 'tel'), 1);
  assert.equal(get(db, d, 'ct_kind', 'out'), 1);
  assert.equal(get(db, d, 'bn_shown'), 1);
  assert.equal(get(db, d, 'bn_accept'), 1);
  assert.equal(get(db, d, 'bn_reject'), undefined);
  assert.equal(db.prepare("SELECT COUNT(*) c FROM daily WHERE day = '2026-10-10'").get().c, 0);
  const before = daily(db);
  runNightly(db, { now: NOW });
  runNightly(db, { now: NOW + H });
  assert.deepEqual(daily(db), before);
});

test('DST: 23:30 UTC im Sommer gehört zum nächsten Berlin-Tag', () => {
  const { db, ev } = mk();
  ev({ ts: Date.UTC(2026, 6, 14, 23, 30), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); // 15.7. 01:30 Berlin
  ev({ ts: Date.UTC(2026, 6, 14, 21, 59), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); // 14.7. 23:59 Berlin
  assert.equal(dayOf(Date.UTC(2026, 6, 14, 23, 30)), '2026-07-15');
  runNightly(db, { now: Date.UTC(2026, 6, 16, 10) });
  assert.equal(get(db, '2026-07-14', 'pv_total'), 1);
  assert.equal(get(db, '2026-07-15', 'pv_total'), 1);
  // Winter: 23:30 UTC = 00:30 Berlin am Folgetag
  assert.equal(dayOf(Date.UTC(2026, 0, 14, 23, 30)), '2026-01-15');
});

test('Aufbewahrung: Rohdaten und Klicks > 30 Tage weg, vorher verdichtet', () => {
  const { db, ev } = mk();
  ev({ ts: NOW - 31 * D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
  ev({ ts: NOW - 29 * D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
  const ins = db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,0.5,0.5)');
  ins.run(KEY, NOW - 31 * D, '/', 'd'); ins.run(KEY, NOW - 29 * D, '/', 'd');
  runNightly(db, { now: NOW });
  assert.equal(db.prepare('SELECT COUNT(*) c FROM events_raw').get().c, 1);
  assert.equal(db.prepare('SELECT COUNT(*) c FROM clicks').get().c, 1);
  assert.equal(get(db, dayOf(NOW - 31 * D), 'pv_total'), 1); // in daily erhalten
});

test('Website mit offSince > 60 Tage wird komplett gelöscht', () => {
  const { db, ev } = mk();
  db.prepare('INSERT INTO sites (siteKey, projectId, domains, active, offSince) VALUES (?,?,?,0,?)')
    .run(OTHER, 'q', '["b.de"]', new Date(NOW - 61 * D).toISOString());
  db.prepare('INSERT INTO sites (siteKey, projectId, domains, active, offSince) VALUES (?,?,?,0,?)')
    .run('young0123456789abcdef', 'r', '["c.de"]', new Date(NOW - 59 * D).toISOString());
  ev({ site: OTHER, ts: NOW - D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
  ev({ ts: NOW - D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
  db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y) VALUES (?,?,?,?,0.1,0.1)').run(OTHER, NOW - D, '/', 'd');
  runNightly(db, { now: NOW });
  for (const t of ['sites', 'events_raw', 'clicks', 'daily', 'agg_state']) {
    const col = t === 'sites' ? 'siteKey' : 'site';
    assert.equal(db.prepare(`SELECT COUNT(*) c FROM ${t} WHERE ${col} = ?`).get(OTHER).c, 0, t);
  }
  assert.equal(db.prepare('SELECT COUNT(*) c FROM sites').get().c, 2); // KEY + junge
  assert.ok(get(db, dayOf(NOW - D), 'pv_total'));
});

test('nextRunAt: 02:30 Berlin (Sommer/Winter)', () => {
  assert.equal(new Date(nextRunAt(Date.UTC(2026, 9, 1, 10))).toISOString(), '2026-10-02T00:30:00.000Z');
  assert.equal(new Date(nextRunAt(Date.UTC(2026, 9, 1, 0, 0))).toISOString(), '2026-10-01T00:30:00.000Z');
  assert.equal(new Date(nextRunAt(Date.UTC(2026, 0, 1, 12))).toISOString(), '2026-01-02T01:30:00.000Z');
});

async function admin(db, now = () => NOW) {
  const server = createAdminServer({ config: { adminToken: 'tok' }, db, now });
  await new Promise((r) => server.listen(0, '127.0.0.1', r));
  const base = `http://127.0.0.1:${server.address().port}`;
  const get = (p, auth = 'Bearer tok') => fetch(base + p, { headers: auth ? { Authorization: auth } : {} });
  return { get, close: () => server.close() };
}

test('Stats-Endpunkt: Form, Serie lückenlos, heute live + Vergangenheit aus daily, consentRate', async () => {
  const { db, ev } = mk();
  const y = NOW - D;
  for (let i = 0; i < 3; i++) ev({ ts: y + i, type: 'pv', path: '/a', vp: 'd', ref: 'g.de', country: 'DE', device: 'd' });
  ev({ ts: y + 10, type: 'en', path: '/a', vp: 'd', visitor: 'v1', sec: 5, scroll: 5 });
  ev({ ts: y + 11, type: 'bn', kind: 'accept' });
  runNightly(db, { now: NOW }); // gestern -> daily
  ev({ ts: NOW - H, type: 'pv', path: '/b', vp: 'm', country: 'AT', device: 'm' }); // heute live
  ev({ ts: NOW - H, type: 'ct', path: '/b', visitor: 'v9', kind: 'mail' });
  ev({ ts: NOW - H, type: 'bn', kind: 'accept' });
  ev({ ts: NOW - H, type: 'bn', kind: 'reject' });
  ev({ ts: NOW - H, type: 'bn', kind: 'reject' });
  ev({ ts: NOW - H, type: 'bn', kind: 'shown' });
  const a = await admin(db);
  const r = await a.get(`/a/admin/stats/${KEY}?range=7`);
  assert.equal(r.status, 200);
  const j = await r.json();
  assert.deepEqual(Object.keys(j).sort(), ['banner', 'contacts', 'countries', 'devices', 'firstDay', 'from', 'pages', 'previous', 'range', 'refs', 'series', 'to', 'totals']);
  assert.equal(j.series.length, 7);
  assert.equal(j.series[0].day, '2026-10-04');
  assert.equal(j.series[6].day, '2026-10-10');
  assert.deepEqual(j.series.map((s) => s.pv), [0, 0, 0, 0, 0, 3, 1]);
  assert.deepEqual(j.totals, { pv: 4, visitors: 2, contacts: 1, consentRate: 2 / 4, engaged: 1, avgSec: 5, avgScroll: 5 });
  assert.deepEqual(j.series[5], { day: '2026-10-09', pv: 3, visitors: 1, contacts: 0, avgSec: 5 });
  assert.deepEqual(j.series[6], { day: '2026-10-10', pv: 1, visitors: 1, contacts: 1, avgSec: null });
  assert.equal(j.from, '2026-10-04'); assert.equal(j.to, '2026-10-10'); assert.equal(j.range, 7);
  assert.deepEqual(j.pages, [{ key: '/a', count: 3 }, { key: '/b', count: 1 }]);
  assert.deepEqual(j.refs[0], { key: 'g.de', count: 3 });
  assert.deepEqual(j.countries, [{ key: 'DE', count: 3 }, { key: 'AT', count: 1 }]);
  assert.deepEqual(j.devices, [{ key: 'd', count: 3 }, { key: 'm', count: 1 }]);
  assert.deepEqual(j.contacts, [{ key: 'mail', count: 1 }]);
  assert.deepEqual(j.banner, { shown: 1, accept: 2, reject: 2 });
  // range 90: 90 Tage
  const j90 = await (await a.get(`/a/admin/stats/${KEY}?range=90`)).json();
  assert.equal(j90.series.length, 90);
  assert.equal(j90.totals.pv, 4);
  a.close();
});

test('Stats: gestern noch nicht verdichtet wird live gezählt, nie doppelt', async () => {
  const { db, ev } = mk();
  ev({ ts: NOW - D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
  const a = await admin(db);
  assert.equal((await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json()).totals.pv, 1);
  runNightly(db, { now: NOW });
  assert.equal((await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json()).totals.pv, 1);
  a.close();
});

test('Stats: Vorzeitraum = die range Kalendertage davor, complete nach firstDay', async () => {
  const { db, ev } = mk();
  // Vorzeitraum zu range 7 am 10.10.: 27.09.–03.10.
  ev({ ts: Date.UTC(2026, 8, 30, 10), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
  ev({ ts: Date.UTC(2026, 9, 1, 10), type: 'en', path: '/', vp: 'd', visitor: 'v1', sec: 20, scroll: 40 });
  ev({ ts: Date.UTC(2026, 9, 2, 10), type: 'ct', path: '/', visitor: 'v2', kind: 'tel' });
  const a = await admin(db);
  let j = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json();
  assert.equal(j.firstDay, '2026-09-30');
  assert.equal(j.previous.complete, false);
  // 26.09. liegt vor dem Vorzeitraum und darf nicht mitzählen
  ev({ ts: Date.UTC(2026, 8, 26, 10), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
  j = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json();
  assert.equal(j.firstDay, '2026-09-26');
  assert.equal(j.previous.complete, true);
  assert.equal(j.previous.from, '2026-09-27'); assert.equal(j.previous.to, '2026-10-03');
  assert.equal(j.previous.series.length, 7); assert.equal(j.previous.series[3].day, '2026-09-30');
  assert.equal(j.previous.totals.pv, 1);
  assert.equal(j.previous.totals.engaged, 1);
  assert.equal(j.previous.totals.avgSec, 20);
  assert.equal(j.previous.totals.contacts, 1);
  assert.deepEqual(j.previous.series[4], { day: '2026-10-01', pv: 0, visitors: 1, contacts: 0, avgSec: 20 });
  assert.deepEqual(j.previous.contacts, [{ key: 'tel', count: 1 }]);
  assert.equal(j.totals.pv, 0); // aktueller Zeitraum leer
  assert.equal(j.totals.avgSec, null);
  a.close();
});

test('Stats: firstDay aus daily und Rohdaten, leer ohne Daten', async () => {
  const { db, ev } = mk();
  const a = await admin(db);
  const first = async () => (await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json());
  let j = await first();
  assert.equal(j.firstDay, '');
  assert.equal(j.previous.complete, false);
  // nur Rohdaten (heute)
  ev({ ts: NOW - H, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
  assert.equal((await first()).firstDay, '2026-10-10');
  // 40 Tage alt: nach dem Nachtlauf nur noch in daily (Rohdaten > 30 Tage gelöscht)
  ev({ ts: NOW - 40 * D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
  assert.equal((await first()).firstDay, '2026-08-31');
  runNightly(db, { now: NOW });
  assert.equal(db.prepare('SELECT COUNT(*) c FROM events_raw WHERE ts < ?').get(NOW - 31 * D).c, 0);
  j = await first();
  assert.equal(j.firstDay, '2026-08-31');
  assert.equal(j.previous.complete, true);
  a.close();
});

test('Stats: avgSec/avgScroll gerundet, null ohne en; Tageswert avgSec', async () => {
  const { db, ev } = mk();
  const t = NOW - D; // 9.10.
  ev({ ts: t, type: 'en', path: '/', vp: 'd', visitor: 'v1', sec: 30, scroll: 50 });
  ev({ ts: t + 1, type: 'en', path: '/a', vp: 'd', visitor: 'v2', sec: 91, scroll: 100 });
  const a = await admin(db);
  const j = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json();
  assert.equal(j.totals.engaged, 2);
  assert.equal(j.totals.avgSec, 61); // 60,5 gerundet
  assert.equal(j.totals.avgScroll, 75);
  assert.equal(j.series[5].avgSec, 61);
  assert.equal(j.series[4].avgSec, null);
  // ohne en: beide null, engaged 0
  const leer = await (await a.get(`/a/admin/stats/${KEY}?range=30`)).json();
  assert.equal(leer.previous.totals.engaged, 0);
  assert.equal(leer.previous.totals.avgSec, null);
  assert.equal(leer.previous.totals.avgScroll, null);
  a.close();
});

test('Stats: Vorzeitraum über die Zeitumstellung lückenlos und ohne Überlappung', async () => {
  const NOV = Date.UTC(2026, 10, 10, 10); // range 30, Zeitumstellung am 25.10. liegt im Zeitraum
  const { db } = mk();
  const a = await admin(db, () => NOV);
  const j = await (await a.get(`/a/admin/stats/${KEY}?range=30`)).json();
  assert.equal(j.series[0].day, '2026-10-12');
  assert.equal(j.series[29].day, '2026-11-10');
  assert.equal(j.previous.series[29].day, '2026-10-11');
  assert.equal(j.previous.series[0].day, '2026-09-12');
  assert.equal(j.previous.from, '2026-09-12'); assert.equal(j.previous.to, '2026-10-11');
  const all = [...j.series, ...j.previous.series].map((s) => s.day);
  assert.equal(all.length, 60);
  assert.equal(new Set(all).size, 60);
  a.close();
});

test('Stats: Vorzeitraum aus daily und live, nie doppelt', async () => {
  const { db, ev } = mk();
  ev({ ts: Date.UTC(2026, 9, 1, 10), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' });
  ev({ ts: Date.UTC(2026, 9, 1, 11), type: 'ct', path: '/', visitor: 'v1', kind: 'mail' });
  const a = await admin(db);
  const get = async () => (await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json()).previous;
  const live = await get(); // noch nicht verdichtet: live aus den Rohdaten
  assert.equal(live.totals.pv, 1);
  runNightly(db, { now: NOW }); // jetzt aus daily
  const verdichtet = await get();
  assert.equal(verdichtet.totals.pv, 1);
  assert.deepEqual(verdichtet, live);
  runNightly(db, { now: NOW + H });
  assert.deepEqual(await get(), live);
  a.close();
});

test('Stats: Top 20 absteigend, consentRate 0 ohne Entscheidungen, Fehlerfälle', async () => {
  const { db, ev } = mk();
  for (let p = 0; p < 25; p++) for (let i = 0; i <= p; i++) ev({ ts: NOW - 1000, type: 'pv', path: `/p${p}`, vp: 'd', country: 'DE', device: 'd' });
  const a = await admin(db);
  const j = await (await a.get(`/a/admin/stats/${KEY}?range=30`)).json();
  assert.equal(j.pages.length, 20);
  assert.equal(j.pages[0].key, '/p24');
  assert.ok(j.pages.every((e, i, arr) => !i || arr[i - 1].count >= e.count));
  assert.equal(j.totals.consentRate, 0);
  assert.equal((await a.get(`/a/admin/stats/${KEY}?range=5`)).status, 400);
  assert.equal((await a.get(`/a/admin/stats/${KEY}?range=abc`)).status, 400);
  assert.equal((await a.get(`/a/admin/stats/${OTHER}?range=7`)).status, 404);
  assert.equal((await a.get(`/a/admin/stats/${KEY}?range=7`, null)).status, 401);
  assert.equal((await a.get(`/a/admin/stats/${KEY}?range=7`, 'Bearer nope')).status, 401);
  a.close();
});

test('scheduleNightly: früh feuernder Timer läuft nicht vorzeitig und nie doppelt für dasselbe 02:30', () => {
  let t = Date.UTC(2026, 9, 10, 10, 0, 0);
  const pending = [];
  const timers = { setTimeout: (fn, ms) => { const h = { fn, at: t + ms }; pending.push(h); return h; }, clearTimeout: (h) => { const i = pending.indexOf(h); if (i >= 0) pending.splice(i, 1); } };
  const runs = [];
  const quiet = { log() {}, error() {} };
  const s = scheduleNightly({}, { startDelayMs: 1000, log: quiet, now: () => t, timers, job: (db, o) => { runs.push(o.now); return { aggregated: 0, sitesDeleted: 0 }; } });
  const fire = (clock) => { const h = pending.shift(); t = clock ?? h.at; h.fn(); };
  fire(); // Startlauf
  assert.equal(runs.length, 1);
  const target = nextRunAt(t);
  assert.equal(pending.length, 1);
  assert.equal(pending[0].at, target);
  fire(target - 500); // Timer feuert 500 ms zu früh
  assert.equal(runs.length, 1, 'kein vorzeitiger Lauf');
  assert.equal(pending.length, 1);
  assert.equal(pending[0].at, target, 'neu auf das Ziel gesetzt');
  fire(target); // pünktlich
  assert.equal(runs.length, 2);
  assert.equal(pending[0].at, nextRunAt(target + 1000));
  assert.ok(pending[0].at > target + 20 * 3600000, 'nächster Lauf am Folgetag');
  fire(pending[0].at - 1); // wieder zu früh
  assert.equal(runs.length, 2);
  s.stop();
  assert.equal(pending.length, 0);
});

test('T2: pv ohne Einwilligung + cx: pv_total 1, pv_ref nur {google.com:1}; cx zählt nie pv_total', () => {
  const { db, ev } = mk();
  const t = NOW - D;
  ev({ ts: t, type: 'pv', path: '/', vp: null, ref: null, country: 'DE', device: 'd' });
  ev({ ts: t, type: 'cx', vp: 'd', ref: 'google.com' });
  ev({ ts: t, type: 'cx', vp: 'd', ref: null });
  ev({ ts: t, type: 'pv', path: '/', vp: 'd', ref: null, country: 'DE', device: 'd' });
  runNightly(db, { now: NOW });
  const d = '2026-10-09';
  assert.equal(get(db, d, 'pv_total'), 2);
  assert.equal(get(db, d, 'pv_ref', 'google.com'), 1);
  assert.equal(get(db, d, 'pv_ref', ''), 2); // pv mit vp und cx ohne ref
  const refs = db.prepare("SELECT key FROM daily WHERE site=? AND dim='pv_ref' ORDER BY key").all(KEY).map((r) => r.key);
  assert.deepEqual(refs, ['', 'google.com']);
});

test('T2: Stats-Endpunkt refs live und aus daily, nie doppelt', async () => {
  const { db, ev } = mk();
  ev({ ts: NOW - D, type: 'pv', path: '/', vp: null, country: 'DE', device: 'd' });
  ev({ ts: NOW - D, type: 'cx', vp: 'd', ref: 'google.com' });
  const a = await admin(db);
  const live = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json();
  assert.equal(live.totals.pv, 1);
  assert.deepEqual(live.refs, [{ key: 'google.com', count: 1 }]);
  runNightly(db, { now: NOW });
  const agg = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json();
  assert.equal(agg.totals.pv, 1);
  assert.deepEqual(agg.refs, [{ key: 'google.com', count: 1 }]);
  a.close();
});