Analytics: Laenderzuordnung

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

analytics/scripts/geo-update.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+# Lädt die aktuelle DB-IP-Lite-Länderdatei (monatlich) nach $ANALYTICS_DATA_DIR/geo.csv und startet
+# den Dienst nur neu, wenn sich die Datei geändert hat. Läuft auf dem Host (z. B. per Cron, 1x pro Woche).
+# Die alte geo.csv wird nie gelöscht, bevor die neue geprüft und umbenannt ist.
+set -eu
+
+DIR="${ANALYTICS_DATA_DIR:-/srv/webklar-analytics}"
+CONTAINER="${ANALYTICS_CONTAINER:-webklar-analytics}"
+NEU="$DIR/geo.csv.new"
+ZIEL="$DIR/geo.csv"
+
+fehler() { echo "Fehler (geo-update): $*" >&2; rm -f "$NEU" "$NEU.gz"; exit 1; }
+
+command -v curl >/dev/null 2>&1 || fehler "curl fehlt."
+command -v gunzip >/dev/null 2>&1 || fehler "gunzip fehlt."
+[ -d "$DIR" ] || fehler "Datenverzeichnis $DIR existiert nicht."
+
+JETZT="$(date -u +%Y-%m)"
+JAHR="${JETZT%-*}"
+MONAT="${JETZT#*-}"
+m="$(printf '%s' "$MONAT" | sed 's/^0//')"
+if [ "$m" -eq 1 ]; then VORMONAT="$((JAHR - 1))-12"; else VORMONAT="$JAHR-$(printf '%02d' "$((m - 1))")"; fi
+
+geladen=""
+for monat in "$JETZT" "$VORMONAT"; do
+  url="https://download.db-ip.com/free/dbip-country-lite-$monat.csv.gz"
+  if curl -fsSL --retry 2 --max-time 300 -o "$NEU.gz" "$url" 2>/dev/null && gunzip -c "$NEU.gz" > "$NEU" 2>/dev/null; then
+    geladen="$monat"
+    break
+  fi
+done
+rm -f "$NEU.gz"
+[ -n "$geladen" ] || fehler "Download von DB-IP fehlgeschlagen (weder $JETZT noch $VORMONAT). Bestehende geo.csv bleibt unverändert."
+
+zeilen="$(wc -l < "$NEU" | tr -d ' ')"
+[ "$zeilen" -gt 100000 ] || fehler "Neue Datei hat nur $zeilen Zeilen (erwartet > 100000). Bestehende geo.csv bleibt unverändert."
+grep -Eq ',"?DE"?\r?$' "$NEU" || fehler "Neue Datei enthält kein 'DE'. Bestehende geo.csv bleibt unverändert."
+
+if [ -f "$ZIEL" ] && cmp -s "$NEU" "$ZIEL"; then
+  rm -f "$NEU"
+  echo "geo.csv ($geladen) unverändert, kein Neustart nötig."
+  exit 0
+fi
+
+mv -f "$NEU" "$ZIEL"
+echo "geo.csv aktualisiert ($geladen, $zeilen Zeilen)."
+docker restart "$CONTAINER" >/dev/null || fehler "Neustart von $CONTAINER fehlgeschlagen (geo.csv ist aber aktualisiert)."
+echo "$CONTAINER neu gestartet."

analytics/src/geo.js

@@ -0,0 +1,136 @@
+// Länderzuordnung per DB-IP Lite (country, CSV: start_ip,end_ip,land). Die Datei wird einmal beim
+// Start geladen (IPv4 als Uint32, IPv6 als 2×64 Bit in BigUint64Array) und per Binärsuche abgefragt.
+import { readFileSync } from 'node:fs';
+import net from 'node:net';
+
+const MASK64 = (1n << 64n) - 1n;
+
+function parseV4(s) {
+  const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(s);
+  if (!m) return -1;
+  let n = 0;
+  for (let i = 1; i <= 4; i++) {
+    const o = Number(m[i]);
+    if (o > 255) return -1;
+    n = n * 256 + o;
+  }
+  return n;
+}
+
+// IPv6 ⇒ 128-Bit-BigInt oder null (auch mit angehängter IPv4-Schreibweise).
+function parseV6(s) {
+  if (net.isIPv6(s) === false) return null;
+  s = s.split('%')[0];
+  const dot = s.lastIndexOf('.');
+  if (dot !== -1) {
+    const v4 = parseV4(s.slice(s.lastIndexOf(':') + 1));
+    if (v4 < 0) return null;
+    s = s.slice(0, s.lastIndexOf(':') + 1) + Math.floor(v4 / 65536).toString(16) + ':' + (v4 % 65536).toString(16);
+  }
+  const [head, tail] = s.split('::');
+  const h = head ? head.split(':') : [];
+  const t = tail === undefined ? [] : (tail ? tail.split(':') : []);
+  const fill = tail === undefined ? 0 : 8 - h.length - t.length;
+  const groups = [...h, ...Array(Math.max(fill, 0)).fill('0'), ...t];
+  if (groups.length !== 8) return null;
+  let n = 0n;
+  for (const g of groups) n = (n << 16n) | BigInt(parseInt(g, 16));
+  return n;
+}
+
+const pack = (c) => (c.length === 2 && c !== 'ZZ' && /^[A-Za-z]{2}$/.test(c) ? (c.toUpperCase().charCodeAt(0) << 8) | c.toUpperCase().charCodeAt(1) : 0);
+const unpack = (n) => (n ? String.fromCharCode(n >> 8, n & 255) : '');
+
+function sortedBy(n, cmp) {
+  for (let i = 1; i < n; i++) if (cmp(i - 1, i) > 0) return false;
+  return true;
+}
+
+export function loadGeo(file) {
+  const none = { lookup: () => '', size: 0 };
+  let text;
+  try { text = readFileSync(file, 'utf8'); } catch (e) {
+    console.warn(`Analytics: Länderdatei ${file} nicht lesbar (${e.code || e.message}); Länder bleiben leer.`);
+    return none;
+  }
+  // Obergrenze der Zeilen für die Vorab-Reservierung
+  let cap = 1;
+  for (let i = text.indexOf('\n'); i !== -1; i = text.indexOf('\n', i + 1)) cap++;
+  let n4 = 0;
+  let n6 = 0;
+  const s4 = new Uint32Array(cap); const e4 = new Uint32Array(cap); const c4 = new Uint16Array(cap);
+  const sh = new BigUint64Array(cap); const sl = new BigUint64Array(cap);
+  const eh = new BigUint64Array(cap); const el = new BigUint64Array(cap); const c6 = new Uint16Array(cap);
+  let pos = 0;
+  while (pos < text.length) {
+    let nl = text.indexOf('\n', pos);
+    if (nl === -1) nl = text.length;
+    const line = text.slice(pos, nl).replace(/["\r]/g, '').trim();
+    pos = nl + 1;
+    if (!line) continue;
+    const p = line.split(',');
+    if (p.length < 3) continue;
+    const a = p[0].trim(); const b = p[1].trim(); const c = pack(p[2].trim());
+    if (a.includes(':')) {
+      const x = parseV6(a); const y = parseV6(b);
+      if (x === null || y === null || y < x) continue;
+      sh[n6] = x >> 64n; sl[n6] = x & MASK64; eh[n6] = y >> 64n; el[n6] = y & MASK64; c6[n6++] = c;
+    } else {
+      const x = parseV4(a); const y = parseV4(b);
+      if (x < 0 || y < x) continue;
+      s4[n4] = x; e4[n4] = y; c4[n4++] = c;
+    }
+  }
+  text = null;
+
+  // Falls die Datei nicht sortiert ist: Indexsortierung
+  let S4 = s4.slice(0, n4); let E4 = e4.slice(0, n4); let C4 = c4.slice(0, n4);
+  if (!sortedBy(n4, (i, j) => S4[i] - S4[j])) {
+    const idx = Array.from({ length: n4 }, (_, i) => i).sort((i, j) => S4[i] - S4[j]);
+    S4 = Uint32Array.from(idx, (i) => s4[i]); E4 = Uint32Array.from(idx, (i) => e4[i]); C4 = Uint16Array.from(idx, (i) => c4[i]);
+  }
+  let SH = sh.slice(0, n6); let SL = sl.slice(0, n6); let EH = eh.slice(0, n6); let EL = el.slice(0, n6); let C6 = c6.slice(0, n6);
+  const cmp6 = (i, j) => (SH[i] !== SH[j] ? (SH[i] < SH[j] ? -1 : 1) : (SL[i] === SL[j] ? 0 : (SL[i] < SL[j] ? -1 : 1)));
+  if (!sortedBy(n6, cmp6)) {
+    const idx = Array.from({ length: n6 }, (_, i) => i).sort(cmp6);
+    const pick = (A, T) => T.from(idx, (i) => A[i]);
+    const [a, b, c, d, e] = [pick(SH, BigUint64Array), pick(SL, BigUint64Array), pick(EH, BigUint64Array), pick(EL, BigUint64Array), pick(C6, Uint16Array)];
+    SH = a; SL = b; EH = c; EL = d; C6 = e;
+  }
+
+  const look4 = (v) => {
+    let lo = 0; let hi = n4 - 1; let best = -1;
+    while (lo <= hi) {
+      const mid = (lo + hi) >>> 1;
+      if (S4[mid] <= v) { best = mid; lo = mid + 1; } else hi = mid - 1;
+    }
+    return best >= 0 && v <= E4[best] ? unpack(C4[best]) : '';
+  };
+  const look6 = (h, l) => {
+    let lo = 0; let hi = n6 - 1; let best = -1;
+    while (lo <= hi) {
+      const mid = (lo + hi) >>> 1;
+      if (SH[mid] < h || (SH[mid] === h && SL[mid] <= l)) { best = mid; lo = mid + 1; } else hi = mid - 1;
+    }
+    if (best < 0) return '';
+    const inside = h < EH[best] || (h === EH[best] && l <= EL[best]);
+    return inside ? unpack(C6[best]) : '';
+  };
+
+  return {
+    size: n4 + n6,
+    lookup(ip) {
+      if (typeof ip !== 'string') return '';
+      ip = ip.trim();
+      if (!ip.includes(':')) {
+        const v = parseV4(ip);
+        return v < 0 ? '' : look4(v);
+      }
+      const v = parseV6(ip);
+      if (v === null) return '';
+      const h = v >> 64n; const l = v & MASK64;
+      if (h === 0n && (l >> 32n) === 0xffffn) return look4(Number(l & 0xffffffffn)); // ::ffff:a.b.c.d
+      return look6(h, l);
+    },
+  };
+}

analytics/src/index.js

@@ -3,13 +3,14 @@ import { join } from 'node:path';
 import { loadConfig } from './config.js';
 import { openDb } from './db.js';
 import { createSalt } from './salt.js';
+import { loadGeo } from './geo.js';
 import { createPublicServer, createAdminServer } from './server.js';
 
-// Platzhalter bis Task 3 (loadGeo aus config.geoFile): kein Land.
-const geo = { lookup: () => '' };
 const config = loadConfig(process.env);
 mkdirSync(config.dataDir, { recursive: true });
 const db = openDb(join(config.dataDir, 'analytics.db'));
+const geo = loadGeo(config.geoFile || join(config.dataDir, 'geo.csv'));
+if (geo.size) console.log(`Analytics: ${geo.size} Länder-Bereiche geladen`);
 
 const servers = [
   ['öffentlich', createPublicServer({ config, db, geo, salt: createSalt() }), config.port],

analytics/test/geo.test.js

@@ -0,0 +1,110 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { loadGeo } from '../src/geo.js';
+
+const CSV = [
+  '1.0.0.0,1.0.0.255,AU',
+  '5.9.0.0,5.9.255.255,DE',
+  '5.10.0.0,5.10.0.9,ZZ',
+  '8.8.8.0,8.8.8.255,US',
+  '255.255.255.0,255.255.255.255,FR',
+  '2001:db8::,2001:db8:ffff:ffff:ffff:ffff:ffff:ffff,AT',
+  '2a01:4f8::,2a01:4f8:ffff:ffff:ffff:ffff:ffff:ffff,DE',
+  '2a02:1::1,2a02:1::ff,CH',
+  '',
+].join('\n');
+
+function fixture(content) {
+  const dir = mkdtempSync(join(tmpdir(), 'geo-'));
+  const f = join(dir, 'geo.csv');
+  writeFileSync(f, content);
+  return f;
+}
+
+test('IPv4: Treffer und Grenzen', () => {
+  const g = loadGeo(fixture(CSV));
+  assert.equal(g.size, 8);
+  assert.equal(g.lookup('5.9.0.0'), 'DE');
+  assert.equal(g.lookup('5.9.123.4'), 'DE');
+  assert.equal(g.lookup('5.9.255.255'), 'DE');
+  assert.equal(g.lookup('5.8.255.255'), '');
+  assert.equal(g.lookup('5.10.0.0'), '');
+  assert.equal(g.lookup('1.0.0.255'), 'AU');
+  assert.equal(g.lookup('1.0.1.0'), '');
+  assert.equal(g.lookup('0.0.0.1'), '');
+  assert.equal(g.lookup('255.255.255.255'), 'FR');
+  assert.equal(g.lookup('8.8.8.8'), 'US');
+});
+
+test('IPv6: Treffer, Grenzen, Kurzschreibweise', () => {
+  const g = loadGeo(fixture(CSV));
+  assert.equal(g.lookup('2a01:4f8::'), 'DE');
+  assert.equal(g.lookup('2a01:4f8:1:2:3:4:5:6'), 'DE');
+  assert.equal(g.lookup('2A01:04F8:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF'), 'DE');
+  assert.equal(g.lookup('2a01:4f9::'), '');
+  assert.equal(g.lookup('2a01:4f7:ffff:ffff:ffff:ffff:ffff:ffff'), '');
+  assert.equal(g.lookup('2001:db8::1'), 'AT');
+  assert.equal(g.lookup('2a02:1::1'), 'CH');
+  assert.equal(g.lookup('2a02:1::ff'), 'CH');
+  assert.equal(g.lookup('2a02:1::100'), '');
+  assert.equal(g.lookup('2a02:1::'), '');
+  assert.equal(g.lookup('::1'), '');
+});
+
+test('IPv4-gemappte IPv6-Adressen zählen als IPv4', () => {
+  const g = loadGeo(fixture(CSV));
+  assert.equal(g.lookup('::ffff:5.9.1.1'), 'DE');
+  assert.equal(g.lookup('::FFFF:8.8.8.8'), 'US');
+  assert.equal(g.lookup('::ffff:808:808'), 'US');
+  assert.equal(g.lookup('::ffff:9.9.9.9'), '');
+});
+
+test('ZZ, ungültige Eingaben, unsortierte Datei', () => {
+  const g = loadGeo(fixture(CSV));
+  assert.equal(g.lookup('5.10.0.5'), '');
+  for (const bad of ['', 'abc', '1.2.3', '256.1.1.1', '1.2.3.4.5', '2001:db8::zz', ':::', '1::2::3', null, undefined, 42]) {
+    assert.equal(g.lookup(bad), '', String(bad));
+  }
+  const u = loadGeo(fixture('8.8.8.0,8.8.8.255,US\n1.0.0.0,1.0.0.255,AU\n2a01:4f8::,2a01:4f8::ff,DE\n2001:db8::,2001:db8::ff,AT\n'));
+  assert.equal(u.lookup('1.0.0.7'), 'AU');
+  assert.equal(u.lookup('8.8.8.8'), 'US');
+  assert.equal(u.lookup('2001:db8::7'), 'AT');
+  assert.equal(u.lookup('2a01:4f8::7'), 'DE');
+});
+
+test('fehlende Datei: alles leer, eine Warnung, kein Absturz', () => {
+  const warns = [];
+  const orig = console.warn;
+  console.warn = (...a) => warns.push(a.join(' '));
+  try {
+    const g = loadGeo('/nicht/vorhanden/geo.csv');
+    assert.equal(g.lookup('5.9.1.1'), '');
+    assert.equal(g.lookup('2a01:4f8::1'), '');
+    assert.equal(g.size, 0);
+  } finally { console.warn = orig; }
+  assert.equal(warns.length, 1);
+  assert.match(warns[0], /nicht lesbar/);
+});
+
+test('600k Zeilen: lädt zügig, Heap bleibt klein', () => {
+  const lines = [];
+  for (let i = 0; i < 400000; i++) lines.push(`${(i * 4096) >>> 24}.${((i * 4096) >>> 16) & 255}.${((i * 4096) >>> 8) & 255}.0,${(i * 4096) >>> 24}.${((i * 4096) >>> 16) & 255}.${((i * 4096 + 255) >>> 8) & 255}.255,${i % 7 ? 'DE' : 'AT'}`);
+  for (let i = 0; i < 200000; i++) lines.push(`2a00:${(i >> 8).toString(16)}:${(i & 255).toString(16)}::,2a00:${(i >> 8).toString(16)}:${(i & 255).toString(16)}:ffff:ffff:ffff:ffff:ffff,${i % 5 ? 'DE' : 'FR'}`);
+  const f = fixture(lines.join('\n'));
+  lines.length = 0;
+  globalThis.gc?.();
+  const before = process.memoryUsage().heapUsed; const rss0 = process.memoryUsage().rss;
+  const t0 = Date.now();
+  const g = loadGeo(f);
+  const ms = Date.now() - t0;
+  globalThis.gc?.();
+  const grow = (process.memoryUsage().heapUsed - before) / 1048576;
+  console.log(`# geo 600k: ${ms} ms, Heap +${grow.toFixed(1)} MB, RSS +${((process.memoryUsage().rss - rss0) / 1048576).toFixed(0)} MB, Zeilen ${g.size}`);
+  assert.equal(g.size, 600000);
+  assert.equal(g.lookup('0.0.16.5'), 'DE');
+  assert.equal(g.lookup('2a00:0:1::5'), 'DE');
+  assert.ok(grow < 100, `Heap +${grow} MB`);
+});