Analytics: Ereignisse annehmen

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

analytics/src/index.js

@@ -2,14 +2,17 @@ import { mkdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { loadConfig } from './config.js';
 import { openDb } from './db.js';
+import { createSalt } from './salt.js';
 import { createPublicServer, createAdminServer } from './server.js';
 
+// Platzhalter bis Task 3 (loadGeo aus config.geoFile): kein Land.
+const geo = { lookup: () => '' };
 const config = loadConfig(process.env);
 mkdirSync(config.dataDir, { recursive: true });
 const db = openDb(join(config.dataDir, 'analytics.db'));
 
 const servers = [
-  ['öffentlich', createPublicServer({ config, db }), config.port],
+  ['öffentlich', createPublicServer({ config, db, geo, salt: createSalt() }), config.port],
   ['intern (Admin)', createAdminServer({ config, db }), config.adminPort],
 ];
 for (const [name, server, port] of servers) {

analytics/src/ingest.js

@@ -0,0 +1,180 @@
+// Annahme von POST /a/e. Datenschutz-Kern: IP und User-Agent leben nur in lokalen
+// Variablen (Geo, Limit-Schlüssel, Besucher-Hash) und werden nie gespeichert oder geloggt.
+import { classifyUa } from './ua.js';
+
+const MAX_BODY = 4096;
+const SITE_KEY = /^[a-z0-9]{16,40}$/;
+const HOST = /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/;
+const LIMIT = 120;
+const HOUR = 3600 * 1000;
+const MINUTE = 60 * 1000;
+const FIELDS = {
+  pv: ['t', 'site', 'path', 'ref', 'vp'],
+  bn: ['t', 'site', 'a'],
+  en: ['t', 'site', 'path', 'vp', 'sec', 'scroll'],
+  ck: ['t', 'site', 'path', 'vp', 'x', 'y', 'sel'],
+  ct: ['t', 'site', 'path', 'kind', 'host'],
+};
+
+function readBody(req) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    let size = 0;
+    let tooBig = false;
+    req.on('data', (c) => {
+      if (tooBig) return;
+      size += c.length;
+      if (size > MAX_BODY) { tooBig = true; chunks.length = 0; return; }
+      chunks.push(c);
+    });
+    req.on('end', () => (tooBig ? reject(Object.assign(new Error('zu groß'), { status: 413 })) : resolve(Buffer.concat(chunks).toString('utf8'))));
+    req.on('error', reject);
+  });
+}
+
+// Rechtester X-Forwarded-For-Eintrag (von Traefik angehängt), sonst Socket-Adresse.
+function clientIp(req) {
+  const xff = String(req.headers['x-forwarded-for'] || '').split(',').map((s) => s.trim()).filter(Boolean);
+  const ip = xff.length ? xff[xff.length - 1] : (req.socket.remoteAddress || '');
+  return ip.replace(/^::ffff:/i, '');
+}
+
+function hostOf(value) {
+  if (typeof value !== 'string' || !value) return '';
+  try { return new URL(value).hostname.toLowerCase(); } catch { return ''; }
+}
+
+function cleanPath(p) {
+  if (typeof p !== 'string') return null;
+  const s = p.split(/[?#]/)[0];
+  if (!s.startsWith('/') || s.length > 300 || /[\u0000-\u001f\u007f]/.test(s)) return null;
+  return s;
+}
+
+function cleanHost(h) {
+  if (typeof h !== 'string') return null;
+  const s = h.toLowerCase();
+  return s.length <= 253 && HOST.test(s) ? s : null;
+}
+
+function cleanRef(ref, domains) {
+  if (typeof ref !== 'string' || !ref) return null;
+  const host = cleanHost(ref.includes('://') ? hostOf(ref) : ref);
+  return host && !domains.includes(host) ? host : null;
+}
+
+const cleanVp = (v) => (v === 'm' || v === 't' || v === 'd' ? v : null);
+const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : null);
+const clamp = (v, lo, hi) => Math.min(hi, Math.max(lo, v));
+
+export function createIngest({ db, geo = { lookup: () => '' }, salt, now = Date.now }) {
+  const getSite = db.prepare('SELECT domains, active FROM sites WHERE siteKey = ?');
+  const activeSites = db.prepare('SELECT domains FROM sites WHERE active = 1');
+  const insEvent = db.prepare(`INSERT INTO events_raw (site, ts, type, path, vp, ref, country, device, visitor, sec, scroll, kind, host)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+  const insClick = db.prepare('INSERT INTO clicks (site, ts, path, vp, x, y, sel) VALUES (?, ?, ?, ?, ?, ?, ?)');
+  const windows = new Map(); // rateKey -> { w, n }; rateKey ist ein HMAC, keine IP
+  let lastPrune = 0;
+
+  const originHost = (req) => hostOf(req.headers.origin) || (req.headers.origin ? '' : hostOf(req.headers.referer));
+  const parseDomains = (s) => { try { return JSON.parse(s); } catch { return []; } };
+
+  function limited(key) {
+    const t = now();
+    const w = Math.floor(t / MINUTE);
+    if (w !== lastPrune) { // alte Fenster entfernen
+      for (const [k, v] of windows) if (v.w !== w) windows.delete(k);
+      lastPrune = w;
+    }
+    let e = windows.get(key);
+    if (!e || e.w !== w) { e = { w, n: 0 }; windows.set(key, e); }
+    return ++e.n > LIMIT;
+  }
+
+  function cors(res, origin) {
+    res.setHeader('Access-Control-Allow-Origin', origin);
+    res.setHeader('Vary', 'Origin');
+  }
+
+  function store(ev, site, domains, req, ip, ua) {
+    const spec = FIELDS[ev.t];
+    if (!spec) return;
+    const e = {};
+    for (const k of spec) if (k in ev) e[k] = ev[k]; // unbekannte Felder werden nie übernommen
+    const t = now();
+    if (e.t === 'bn') {
+      if (!['shown', 'accept', 'reject'].includes(e.a)) return;
+      insEvent.run(site, Math.floor(t / HOUR) * HOUR, 'bn', null, null, null, null, null, null, null, null, e.a, null);
+      return;
+    }
+    const path = cleanPath(e.path);
+    if (!path) return;
+    if (e.t === 'pv') {
+      const vp = cleanVp(e.vp);
+      if (!vp) return;
+      const c = String(geo.lookup(ip) || '');
+      const country = /^[A-Za-z]{2}$/.test(c) ? c.toUpperCase() : '';
+      insEvent.run(site, t, 'pv', path, vp, cleanRef(e.ref, domains), country, classifyUa(ua).device, null, null, null, null, null);
+      return;
+    }
+    if (e.t === 'ck') {
+      const vp = cleanVp(e.vp);
+      const x = num(e.x);
+      const y = num(e.y);
+      if (!vp || x === null || y === null) return;
+      const sel = typeof e.sel === 'string' ? e.sel.slice(0, 200) : null;
+      insClick.run(site, t, path, vp, Math.round(clamp(x, 0, 1) * 1000) / 1000, Math.round(clamp(y, 0, 1) * 1000) / 1000, sel);
+      return;
+    }
+    const visitor = salt.visitorId(site, ip, ua);
+    if (e.t === 'en') {
+      const vp = cleanVp(e.vp);
+      const sec = num(e.sec);
+      const scroll = num(e.scroll);
+      if (!vp || sec === null || scroll === null) return;
+      insEvent.run(site, t, 'en', path, vp, null, null, null, visitor, Math.floor(clamp(sec, 0, 3600)), Math.floor(clamp(scroll, 0, 100)), null, null);
+      return;
+    }
+    if (e.t === 'ct') {
+      if (!['tel', 'mail', 'wa', 'out'].includes(e.kind)) return;
+      let host = null;
+      if (e.kind === 'out') { host = cleanHost(e.host); if (!host) return; }
+      insEvent.run(site, t, 'ct', path, null, null, null, null, visitor, null, null, e.kind, host);
+    }
+  }
+
+  // Liefert true, wenn die Anfrage /a/e betraf.
+  return async function handle(req, res, pathname) {
+    if (pathname !== '/a/e') return false;
+    const done = (status) => { res.writeHead(status); res.end(); return true; };
+    if (req.method === 'OPTIONS') {
+      const host = originHost(req);
+      const origin = req.headers.origin;
+      if (host && origin && activeSites.all().some((s) => parseDomains(s.domains).includes(host))) {
+        cors(res, origin);
+        res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS');
+        res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+        res.setHeader('Access-Control-Max-Age', '86400');
+      }
+      return done(204);
+    }
+    if (req.method !== 'POST') { res.setHeader('Allow', 'POST, OPTIONS'); return done(405); }
+    let raw;
+    try { raw = await readBody(req); } catch (e) { return done(e.status === 413 ? 413 : 204); }
+    let ev;
+    try { ev = JSON.parse(raw); } catch { return done(204); }
+    if (!ev || typeof ev !== 'object' || Array.isArray(ev) || typeof ev.site !== 'string' || !SITE_KEY.test(ev.site)) return done(204);
+    const site = getSite.get(ev.site);
+    if (!site || !site.active) return done(204);
+    const domains = parseDomains(site.domains);
+    const host = originHost(req);
+    if (!host || !domains.includes(host)) return done(204);
+    if (req.headers.origin) cors(res, req.headers.origin);
+    const ua = String(req.headers['user-agent'] || '');
+    if (classifyUa(ua).bot) return done(204);
+    const ip = clientIp(req);
+    if (limited(salt.rateKey(ev.site, ip))) return done(204);
+    try { store(ev, ev.site, domains, req, ip, ua); } catch (e) { console.error('Analytics-Annahme:', e.message); }
+    return done(204);
+  };
+}

analytics/src/salt.js

@@ -0,0 +1,19 @@
+import crypto from 'node:crypto';
+
+const DAY = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Berlin', year: 'numeric', month: '2-digit', day: '2-digit' });
+
+// Tagessalz nur im RAM; neu erzeugt, sobald sich der Kalendertag in Europe/Berlin ändert.
+export function createSalt({ now = Date.now, random = () => crypto.randomBytes(32) } = {}) {
+  let day = null;
+  let salt = null;
+  const current = () => {
+    const d = DAY.format(now());
+    if (d !== day) { day = d; salt = random(); }
+    return salt;
+  };
+  const hmac = (s) => crypto.createHmac('sha256', current()).update(s).digest('hex');
+  return {
+    visitorId: (site, ip, ua) => hmac(`${site}|${ip}|${ua}`).slice(0, 16),
+    rateKey: (site, ip) => hmac(`${site}|${ip}`),
+  };
+}

analytics/src/server.js

@@ -1,5 +1,7 @@
 import http from 'node:http';
 import crypto from 'node:crypto';
+import { createIngest } from './ingest.js';
+import { createSalt } from './salt.js';
 
 const MAX_ADMIN_BODY = 64 * 1024;
 const SITE_KEY = /^[a-z0-9]{16,40}$/;
@@ -53,13 +55,16 @@ function validateSite(b) {
   return { site: { projectId: b.projectId, domains: b.domains, active: b.active } };
 }
 
-function createHandlerServer({ config, db, role }) {
+function createHandlerServer({ config, db, role, geo, salt, now }) {
+  const ingest = role === 'public' ? createIngest({ db, geo, salt: salt || createSalt({ now }), now }) : null;
   return http.createServer(async (req, res) => {
     try {
       const url = new URL(req.url, 'http://x');
       const parts = url.pathname.split('/').filter(Boolean);
       const method = req.method;
 
+      if (ingest && await ingest(req, res, url.pathname)) return;
+
       if (method === 'GET' && url.pathname === '/a/health') return send(res, 200, { ok: true });
 
       if (role === 'admin' && parts[0] === 'a' && parts[1] === 'admin' && parts[2] === 'sites' && parts.length === 4
@@ -102,7 +107,7 @@ function createHandlerServer({ config, db, role }) {
   });
 }
 
-// Öffentlicher Listener: nur /a/health (Erfassung folgt); Admin-Routen gibt es hier nicht.
-export const createPublicServer = ({ config, db }) => createHandlerServer({ config, db, role: 'public' });
+// Öffentlicher Listener: /a/health und /a/e (Erfassung); Admin-Routen gibt es hier nicht.
+export const createPublicServer = ({ config, db, geo, salt, now }) => createHandlerServer({ config, db, role: 'public', geo, salt, now });
 // Interner Listener für die Website-Verwaltung.
 export const createAdminServer = ({ config, db }) => createHandlerServer({ config, db, role: 'admin' });

analytics/src/ua.js

@@ -0,0 +1,10 @@
+// Gröbste Bot- und Gerätetyp-Erkennung; der User-Agent wird danach verworfen.
+const BOT = /bot|crawl|spider|slurp|headless|lighthouse|pingdom|uptime|curl|wget|python|go-http|java\/|facebookexternalhit|preview/i;
+
+export function classifyUa(ua) {
+  const s = typeof ua === 'string' ? ua : '';
+  if (!s || BOT.test(s)) return { bot: true, device: 'd' };
+  if (/Mobi|Android.*Mobile|iPhone/i.test(s)) return { bot: false, device: 'm' };
+  if (/iPad|Tablet|Android(?!.*Mobile)/i.test(s)) return { bot: false, device: 't' };
+  return { bot: false, device: 'd' };
+}