From 37bec17d73cf4f868c80c9c2bfaf397b13b2114d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 19:49:55 +0000 Subject: [PATCH] Analyse: Vorzeitraum, firstDay, Tageswerte und Durchschnitte in der Statistik; activatedAt in der Ansicht Co-Authored-By: Claude Opus 5.5 --- analytics/src/stats.js | 114 +++++++++++++++++++++++++-------- analytics/test/stats.test.js | 109 ++++++++++++++++++++++++++++++- server/routes/funktionen.js | 1 + test/analyseRoutes.test.js | 3 +- test/analyticsContract.test.js | 4 ++ 5 files changed, 202 insertions(+), 29 deletions(-) diff --git a/analytics/src/stats.js b/analytics/src/stats.js index e2033a9..1e72c90 100644 --- a/analytics/src/stats.js +++ b/analytics/src/stats.js @@ -1,69 +1,131 @@ // Statistik für das Portal: Vergangenheit aus `daily`, noch nicht verdichtete Tage live aus den Rohdaten. import { dayOf, aggregateRows } from './jobs.js'; -const DAY_MS = 86400 * 1000; const TOP = 20; -// Liste der Kalendertage (YYYY-MM-DD), oldest first, endend am Berlin-Tag von `now`. -function dayList(now, range) { - const [y, m, d] = dayOf(now).split('-').map(Number); +// Kalendertag (YYYY-MM-DD) um `n` Tage verschoben, rein kalendarisch (UTC-Datumsrechnung, keine Zeitumstellung). +function shiftDay(day, n) { + const [y, m, d] = day.split('-').map(Number); + return new Date(Date.UTC(y, m - 1, d + n)).toISOString().slice(0, 10); +} + +// Liste der Kalendertage (YYYY-MM-DD), oldest first, endend am Tag `endDay`. +function dayList(endDay, range) { const days = []; - for (let i = range - 1; i >= 0; i--) days.push(new Date(Date.UTC(y, m - 1, d - i)).toISOString().slice(0, 10)); + for (let i = range - 1; i >= 0; i--) days.push(shiftDay(endDay, -i)); return days; } const top = (map) => [...map].map(([key, count]) => ({ key, count })).filter((e) => e.count > 0) .sort((a, b) => b.count - a.count || (a.key < b.key ? -1 : 1)).slice(0, TOP); -export function computeStats(db, site, range, now = Date.now()) { - const days = dayList(now, range); +// Sammelt die Zähler für eine zusammenhängende Tagesliste: verdichtete Tage aus `daily`, +// Tage ohne agg_state (heute, ggf. gestern vor 02:30) live aus den Rohdaten. Kein Tag zählt doppelt. +function collect(db, site, days, now) { const first = days[0]; + const last = days[days.length - 1]; const inRange = new Set(days); - const pvByDay = new Map(days.map((d) => [d, 0])); + const perDay = new Map(days.map((d) => [d, { pv: 0, visitors: 0, contacts: 0, en: 0, sec: 0, scroll: 0 }])); const dims = {}; - const bump = (dim, key, n) => { + const apply = (day, dim, key, n) => { + const t = perDay.get(day); + if (dim === 'pv_total') t.pv += n; + else if (dim === 'visitors') t.visitors += n; + else if (dim === 'ct_kind') t.contacts += n; + else if (dim === 'en_count') t.en += n; + else if (dim === 'en_sec_sum') t.sec += n; + else if (dim === 'scroll_sum') t.scroll += n; if (!dims[dim]) dims[dim] = new Map(); dims[dim].set(key, (dims[dim].get(key) || 0) + n); }; - const apply = (day, dim, key, n) => { - if (dim === 'pv_total') pvByDay.set(day, pvByDay.get(day) + n); - bump(dim, key, n); - }; - const done = new Set(db.prepare('SELECT day FROM agg_state WHERE site = ? AND day >= ?').all(site, first).map((r) => r.day)); - for (const r of db.prepare('SELECT day, dim, key, count FROM daily WHERE site = ? AND day >= ?').all(site, first)) { + const done = new Set(db.prepare('SELECT day FROM agg_state WHERE site = ? AND day >= ? AND day <= ?').all(site, first, last).map((r) => r.day)); + for (const r of db.prepare('SELECT day, dim, key, count FROM daily WHERE site = ? AND day >= ? AND day <= ?').all(site, first, last)) { if (inRange.has(r.day)) apply(r.day, r.dim, r.key, r.count); } - // Tage ohne agg_state (heute, ggf. gestern vor 02:30) live aus den Rohdaten + // Ein Berliner Tag beginnt 22:00/23:00 UTC des Vortags: zwei Tage Rand reichen auf beiden Seiten, danach filtert dayOf + const [fy, fm, fd] = first.split('-').map(Number); + const [ly, lm, ld] = last.split('-').map(Number); + const lo = Date.UTC(fy, fm - 1, fd - 2); + const hi = Math.min(now, Date.UTC(ly, lm - 1, ld + 2)); const live = new Map(); - const from = now - (range + 1) * DAY_MS; for (const r of db.prepare('SELECT ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind FROM events_raw WHERE site = ? AND ts >= ? AND ts <= ? ORDER BY ts') - .all(site, from, now)) { + .all(site, lo, hi)) { const d = dayOf(r.ts); if (!inRange.has(d) || done.has(d)) continue; if (!live.has(d)) live.set(d, []); live.get(d).push(r); } for (const [day, rows] of live) for (const [dim, key, n] of aggregateRows(rows)) apply(day, dim, key, n); + return { days, perDay, dims }; +} +const avg = (sum, n) => (n ? Math.round(sum / n) : null); + +// Formt gesammelte Zähler zu banner + totals + series (gleiche Form für aktuellen Zeitraum und Vorzeitraum). +function shape({ days, perDay, dims }) { const sum = (dim) => [...(dims[dim] || new Map()).values()].reduce((a, b) => a + b, 0); const banner = { shown: sum('bn_shown'), accept: sum('bn_accept'), reject: sum('bn_reject') }; const decided = banner.accept + banner.reject; - const contacts = top(dims.ct_kind || new Map()); + const en = sum('en_count'); return { + banner, totals: { pv: sum('pv_total'), // Summe der Tages-Besucher (kein tagesübergreifender Abgleich möglich: Salz wechselt täglich) visitors: sum('visitors'), contacts: sum('ct_kind'), consentRate: decided ? banner.accept / decided : 0, + engaged: en, + avgSec: avg(sum('en_sec_sum'), en), + avgScroll: avg(sum('scroll_sum'), en), + }, + series: days.map((day) => { + const t = perDay.get(day); + return { day, pv: t.pv, visitors: t.visitors, contacts: t.contacts, avgSec: avg(t.sec, t.en) }; + }), + }; +} + +// Frühester Berliner Tag mit Daten dieser Website ('' = keine): kleinster Tag aus daily und events_raw. +function firstDayOf(db, site) { + const fromDaily = db.prepare('SELECT MIN(day) AS d FROM daily WHERE site = ?').get(site)?.d || ''; + const ts = db.prepare('SELECT MIN(ts) AS t FROM events_raw WHERE site = ?').get(site)?.t; + const fromRaw = ts == null ? '' : dayOf(ts); + if (!fromDaily) return fromRaw; + if (!fromRaw) return fromDaily; + return fromDaily < fromRaw ? fromDaily : fromRaw; +} + +export function computeStats(db, site, range, now = Date.now()) { + const days = dayList(dayOf(now), range); + const cur = collect(db, site, days, now); + const { banner, totals, series } = shape(cur); + // Vorzeitraum: die `range` Kalendertage direkt vor dem aktuellen Zeitraum (kalendarisch, nie now - range * 24 h) + const prevDays = dayList(shiftDay(days[0], -1), range); + const prevCollected = collect(db, site, prevDays, now); + const prev = shape(prevCollected); + const firstDay = firstDayOf(db, site); + return { + range, + from: days[0], + to: days[days.length - 1], + firstDay, + totals, + series, + pages: top(cur.dims.pv_path || new Map()), + refs: top(cur.dims.pv_ref || new Map()), + countries: top(cur.dims.pv_country || new Map()), + devices: top(cur.dims.pv_device || new Map()), + contacts: top(cur.dims.ct_kind || new Map()), + banner, + previous: { + from: prevDays[0], + to: prevDays[prevDays.length - 1], + complete: firstDay !== '' && firstDay <= prevDays[0], + totals: prev.totals, + series: prev.series, + contacts: top(prevCollected.dims.ct_kind || new Map()), }, - series: days.map((day) => ({ day, pv: pvByDay.get(day) })), - pages: top(dims.pv_path || new Map()), - refs: top(dims.pv_ref || new Map()), - countries: top(dims.pv_country || new Map()), - devices: top(dims.pv_device || new Map()), - contacts, - banner, }; } diff --git a/analytics/test/stats.test.js b/analytics/test/stats.test.js index a9e07c2..d24d832 100644 --- a/analytics/test/stats.test.js +++ b/analytics/test/stats.test.js @@ -132,12 +132,15 @@ test('Stats-Endpunkt: Form, Serie lückenlos, heute live + Vergangenheit aus dai const r = await a.get(`/a/admin/stats/${KEY}?range=7`); assert.equal(r.status, 200); const j = await r.json(); - assert.deepEqual(Object.keys(j).sort(), ['banner', 'contacts', 'countries', 'devices', 'pages', 'refs', 'series', 'totals']); + assert.deepEqual(Object.keys(j).sort(), ['banner', 'contacts', 'countries', 'devices', 'firstDay', 'from', 'pages', 'previous', 'range', 'refs', 'series', 'to', 'totals']); assert.equal(j.series.length, 7); assert.equal(j.series[0].day, '2026-10-04'); assert.equal(j.series[6].day, '2026-10-10'); assert.deepEqual(j.series.map((s) => s.pv), [0, 0, 0, 0, 0, 3, 1]); - assert.deepEqual(j.totals, { pv: 4, visitors: 2, contacts: 1, consentRate: 2 / 4 }); + assert.deepEqual(j.totals, { pv: 4, visitors: 2, contacts: 1, consentRate: 2 / 4, engaged: 1, avgSec: 5, avgScroll: 5 }); + assert.deepEqual(j.series[5], { day: '2026-10-09', pv: 3, visitors: 1, contacts: 0, avgSec: 5 }); + assert.deepEqual(j.series[6], { day: '2026-10-10', pv: 1, visitors: 1, contacts: 1, avgSec: null }); + assert.equal(j.from, '2026-10-04'); assert.equal(j.to, '2026-10-10'); assert.equal(j.range, 7); assert.deepEqual(j.pages, [{ key: '/a', count: 3 }, { key: '/b', count: 1 }]); assert.deepEqual(j.refs[0], { key: 'g.de', count: 3 }); assert.deepEqual(j.countries, [{ key: 'DE', count: 3 }, { key: 'AT', count: 1 }]); @@ -161,6 +164,108 @@ test('Stats: gestern noch nicht verdichtet wird live gezählt, nie doppelt', asy a.close(); }); +test('Stats: Vorzeitraum = die range Kalendertage davor, complete nach firstDay', async () => { + const { db, ev } = mk(); + // Vorzeitraum zu range 7 am 10.10.: 27.09.–03.10. + ev({ ts: Date.UTC(2026, 8, 30, 10), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); + ev({ ts: Date.UTC(2026, 9, 1, 10), type: 'en', path: '/', vp: 'd', visitor: 'v1', sec: 20, scroll: 40 }); + ev({ ts: Date.UTC(2026, 9, 2, 10), type: 'ct', path: '/', visitor: 'v2', kind: 'tel' }); + const a = await admin(db); + let j = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json(); + assert.equal(j.firstDay, '2026-09-30'); + assert.equal(j.previous.complete, false); + // 26.09. liegt vor dem Vorzeitraum und darf nicht mitzählen + ev({ ts: Date.UTC(2026, 8, 26, 10), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); + j = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json(); + assert.equal(j.firstDay, '2026-09-26'); + assert.equal(j.previous.complete, true); + assert.equal(j.previous.from, '2026-09-27'); assert.equal(j.previous.to, '2026-10-03'); + assert.equal(j.previous.series.length, 7); assert.equal(j.previous.series[3].day, '2026-09-30'); + assert.equal(j.previous.totals.pv, 1); + assert.equal(j.previous.totals.engaged, 1); + assert.equal(j.previous.totals.avgSec, 20); + assert.equal(j.previous.totals.contacts, 1); + assert.deepEqual(j.previous.series[4], { day: '2026-10-01', pv: 0, visitors: 1, contacts: 0, avgSec: 20 }); + assert.deepEqual(j.previous.contacts, [{ key: 'tel', count: 1 }]); + assert.equal(j.totals.pv, 0); // aktueller Zeitraum leer + assert.equal(j.totals.avgSec, null); + a.close(); +}); + +test('Stats: firstDay aus daily und Rohdaten, leer ohne Daten', async () => { + const { db, ev } = mk(); + const a = await admin(db); + const first = async () => (await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json()); + let j = await first(); + assert.equal(j.firstDay, ''); + assert.equal(j.previous.complete, false); + // nur Rohdaten (heute) + ev({ ts: NOW - H, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); + assert.equal((await first()).firstDay, '2026-10-10'); + // 40 Tage alt: nach dem Nachtlauf nur noch in daily (Rohdaten > 30 Tage gelöscht) + ev({ ts: NOW - 40 * D, type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); + assert.equal((await first()).firstDay, '2026-08-31'); + runNightly(db, { now: NOW }); + assert.equal(db.prepare('SELECT COUNT(*) c FROM events_raw WHERE ts < ?').get(NOW - 31 * D).c, 0); + j = await first(); + assert.equal(j.firstDay, '2026-08-31'); + assert.equal(j.previous.complete, true); + a.close(); +}); + +test('Stats: avgSec/avgScroll gerundet, null ohne en; Tageswert avgSec', async () => { + const { db, ev } = mk(); + const t = NOW - D; // 9.10. + ev({ ts: t, type: 'en', path: '/', vp: 'd', visitor: 'v1', sec: 30, scroll: 50 }); + ev({ ts: t + 1, type: 'en', path: '/a', vp: 'd', visitor: 'v2', sec: 91, scroll: 100 }); + const a = await admin(db); + const j = await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json(); + assert.equal(j.totals.engaged, 2); + assert.equal(j.totals.avgSec, 61); // 60,5 gerundet + assert.equal(j.totals.avgScroll, 75); + assert.equal(j.series[5].avgSec, 61); + assert.equal(j.series[4].avgSec, null); + // ohne en: beide null, engaged 0 + const leer = await (await a.get(`/a/admin/stats/${KEY}?range=30`)).json(); + assert.equal(leer.previous.totals.engaged, 0); + assert.equal(leer.previous.totals.avgSec, null); + assert.equal(leer.previous.totals.avgScroll, null); + a.close(); +}); + +test('Stats: Vorzeitraum über die Zeitumstellung lückenlos und ohne Überlappung', async () => { + const NOV = Date.UTC(2026, 10, 10, 10); // range 30, Zeitumstellung am 25.10. liegt im Zeitraum + const { db } = mk(); + const a = await admin(db, () => NOV); + const j = await (await a.get(`/a/admin/stats/${KEY}?range=30`)).json(); + assert.equal(j.series[0].day, '2026-10-12'); + assert.equal(j.series[29].day, '2026-11-10'); + assert.equal(j.previous.series[29].day, '2026-10-11'); + assert.equal(j.previous.series[0].day, '2026-09-12'); + assert.equal(j.previous.from, '2026-09-12'); assert.equal(j.previous.to, '2026-10-11'); + const all = [...j.series, ...j.previous.series].map((s) => s.day); + assert.equal(all.length, 60); + assert.equal(new Set(all).size, 60); + a.close(); +}); + +test('Stats: Vorzeitraum aus daily und live, nie doppelt', async () => { + const { db, ev } = mk(); + ev({ ts: Date.UTC(2026, 9, 1, 10), type: 'pv', path: '/', vp: 'd', country: 'DE', device: 'd' }); + ev({ ts: Date.UTC(2026, 9, 1, 11), type: 'ct', path: '/', visitor: 'v1', kind: 'mail' }); + const a = await admin(db); + const get = async () => (await (await a.get(`/a/admin/stats/${KEY}?range=7`)).json()).previous; + const live = await get(); // noch nicht verdichtet: live aus den Rohdaten + assert.equal(live.totals.pv, 1); + runNightly(db, { now: NOW }); // jetzt aus daily + const verdichtet = await get(); + assert.equal(verdichtet.totals.pv, 1); + assert.deepEqual(verdichtet, live); + runNightly(db, { now: NOW + H }); + assert.deepEqual(await get(), live); + a.close(); +}); + test('Stats: Top 20 absteigend, consentRate 0 ohne Entscheidungen, Fehlerfälle', async () => { const { db, ev } = mk(); for (let p = 0; p < 25; p++) for (let i = 0; i <= p; i++) ev({ ts: NOW - 1000, type: 'pv', path: `/p${p}`, vp: 'd', country: 'DE', device: 'd' }); diff --git a/server/routes/funktionen.js b/server/routes/funktionen.js index 7c0694a..35299fd 100644 --- a/server/routes/funktionen.js +++ b/server/routes/funktionen.js @@ -70,6 +70,7 @@ async function analyseView(customerId, projectId) { avvVersion: config.analyse.avvVersion, legal: legalPaths(config.analyse.avvVersion), controller, + activatedAt: doc?.state === 'active' ? doc.activatedAt || null : null, } } diff --git a/test/analyseRoutes.test.js b/test/analyseRoutes.test.js index 2fabb7b..454f78d 100644 --- a/test/analyseRoutes.test.js +++ b/test/analyseRoutes.test.js @@ -112,7 +112,7 @@ test('GET /api/funktionen: analyse-Zustand je Projekt, mit und ohne Schalter', a const body = await get('/api/funktionen', { cookie }) const p = (id) => body.projects.find((x) => x.projectId === id) assert.equal(p('p1').features.news.state, 'off') - assert.deepEqual(p('p1').analyse, { state: 'off', devMode: false, accentColor: '#111111', eligible: true, reason: null, reasonText: null, avvVersion: config.analyse.avvVersion, legal: legalPaths(config.analyse.avvVersion), controller: { company: 'Kunde Eins', address: '', email: 'k@example.invalid', ok: true, missing: [] } }) + assert.deepEqual(p('p1').analyse, { state: 'off', devMode: false, accentColor: '#111111', eligible: true, reason: null, reasonText: null, avvVersion: config.analyse.avvVersion, legal: legalPaths(config.analyse.avvVersion), controller: { company: 'Kunde Eins', address: '', email: 'k@example.invalid', ok: true, missing: [] }, activatedAt: null }) assert.equal(p('p2').analyse.eligible, false) assert.equal(p('p2').analyse.reason, 'nicht_oeffentlich') assert.equal(p('p2').analyse.reasonText, an.ELIGIBILITY_MESSAGES.nicht_oeffentlich) @@ -133,6 +133,7 @@ test('Aktivieren und Deaktivieren (Kunde)', async () => { const view = (await get('/api/funktionen', { cookie })).projects.find((x) => x.projectId === 'p1').analyse assert.equal(view.state, 'active') assert.equal(view.accentColor, '#112233') + assert.match(view.activatedAt, /^\d{4}-\d{2}-\d{2}T/) const off = await post(`${A}/deactivate`, { projectId: 'p1' }, { cookie }) assert.deepEqual(off.body, { ok: true, state: 'off' }) assert.equal(stops.length, 1) diff --git a/test/analyticsContract.test.js b/test/analyticsContract.test.js index 9d11b23..4df3a50 100644 --- a/test/analyticsContract.test.js +++ b/test/analyticsContract.test.js @@ -44,6 +44,10 @@ test('Vertrag: putSite -> Ereignisse -> getStats -> heatToken -> deleteSite', as assert.equal(stats.totals.contacts, 1) assert.equal(stats.series.length, 7) assert.equal(stats.series.reduce((n, d) => n + d.pv, 0), 3) + assert.equal(stats.range, 7) + assert.equal(stats.previous.series.length, 7) + assert.ok('firstDay' in stats && 'avgSec' in stats.totals) + assert.ok(stats.series.every((d) => 'visitors' in d && 'contacts' in d && 'avgSec' in d)) assert.deepEqual(stats.pages, [{ key: '/', count: 2 }, { key: '/kontakt', count: 1 }]) assert.deepEqual(stats.refs, [{ key: '', count: 2 }, { key: 'google.com', count: 1 }]) assert.equal(stats.banner.accept, 1)