diff --git a/analytics/scripts/geo-update.sh b/analytics/scripts/geo-update.sh new file mode 100755 index 0000000..a0db4a5 --- /dev/null +++ b/analytics/scripts/geo-update.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# Lädt die aktuelle DB-IP-Lite-Länderdatei (monatlich) nach $ANALYTICS_DATA_DIR/geo.csv und startet +# den Dienst nur neu, wenn sich die Datei geändert hat. Läuft auf dem Host (z. B. per Cron, 1x pro Woche). +# Die alte geo.csv wird nie gelöscht, bevor die neue geprüft und umbenannt ist. +set -eu + +DIR="${ANALYTICS_DATA_DIR:-/srv/webklar-analytics}" +CONTAINER="${ANALYTICS_CONTAINER:-webklar-analytics}" +NEU="$DIR/geo.csv.new" +ZIEL="$DIR/geo.csv" + +fehler() { echo "Fehler (geo-update): $*" >&2; rm -f "$NEU" "$NEU.gz"; exit 1; } + +command -v curl >/dev/null 2>&1 || fehler "curl fehlt." +command -v gunzip >/dev/null 2>&1 || fehler "gunzip fehlt." +[ -d "$DIR" ] || fehler "Datenverzeichnis $DIR existiert nicht." + +JETZT="$(date -u +%Y-%m)" +JAHR="${JETZT%-*}" +MONAT="${JETZT#*-}" +m="$(printf '%s' "$MONAT" | sed 's/^0//')" +if [ "$m" -eq 1 ]; then VORMONAT="$((JAHR - 1))-12"; else VORMONAT="$JAHR-$(printf '%02d' "$((m - 1))")"; fi + +geladen="" +for monat in "$JETZT" "$VORMONAT"; do + url="https://download.db-ip.com/free/dbip-country-lite-$monat.csv.gz" + if curl -fsSL --retry 2 --max-time 300 -o "$NEU.gz" "$url" 2>/dev/null && gunzip -c "$NEU.gz" > "$NEU" 2>/dev/null; then + geladen="$monat" + break + fi +done +rm -f "$NEU.gz" +[ -n "$geladen" ] || fehler "Download von DB-IP fehlgeschlagen (weder $JETZT noch $VORMONAT). Bestehende geo.csv bleibt unverändert." + +zeilen="$(wc -l < "$NEU" | tr -d ' ')" +[ "$zeilen" -gt 100000 ] || fehler "Neue Datei hat nur $zeilen Zeilen (erwartet > 100000). Bestehende geo.csv bleibt unverändert." +grep -Eq ',"?DE"?\r?$' "$NEU" || fehler "Neue Datei enthält kein 'DE'. Bestehende geo.csv bleibt unverändert." + +if [ -f "$ZIEL" ] && cmp -s "$NEU" "$ZIEL"; then + rm -f "$NEU" + echo "geo.csv ($geladen) unverändert, kein Neustart nötig." + exit 0 +fi + +mv -f "$NEU" "$ZIEL" +echo "geo.csv aktualisiert ($geladen, $zeilen Zeilen)." +docker restart "$CONTAINER" >/dev/null || fehler "Neustart von $CONTAINER fehlgeschlagen (geo.csv ist aber aktualisiert)." +echo "$CONTAINER neu gestartet." diff --git a/analytics/src/geo.js b/analytics/src/geo.js new file mode 100644 index 0000000..2c0f51a --- /dev/null +++ b/analytics/src/geo.js @@ -0,0 +1,136 @@ +// Länderzuordnung per DB-IP Lite (country, CSV: start_ip,end_ip,land). Die Datei wird einmal beim +// Start geladen (IPv4 als Uint32, IPv6 als 2×64 Bit in BigUint64Array) und per Binärsuche abgefragt. +import { readFileSync } from 'node:fs'; +import net from 'node:net'; + +const MASK64 = (1n << 64n) - 1n; + +function parseV4(s) { + const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(s); + if (!m) return -1; + let n = 0; + for (let i = 1; i <= 4; i++) { + const o = Number(m[i]); + if (o > 255) return -1; + n = n * 256 + o; + } + return n; +} + +// IPv6 ⇒ 128-Bit-BigInt oder null (auch mit angehängter IPv4-Schreibweise). +function parseV6(s) { + if (net.isIPv6(s) === false) return null; + s = s.split('%')[0]; + const dot = s.lastIndexOf('.'); + if (dot !== -1) { + const v4 = parseV4(s.slice(s.lastIndexOf(':') + 1)); + if (v4 < 0) return null; + s = s.slice(0, s.lastIndexOf(':') + 1) + Math.floor(v4 / 65536).toString(16) + ':' + (v4 % 65536).toString(16); + } + const [head, tail] = s.split('::'); + const h = head ? head.split(':') : []; + const t = tail === undefined ? [] : (tail ? tail.split(':') : []); + const fill = tail === undefined ? 0 : 8 - h.length - t.length; + const groups = [...h, ...Array(Math.max(fill, 0)).fill('0'), ...t]; + if (groups.length !== 8) return null; + let n = 0n; + for (const g of groups) n = (n << 16n) | BigInt(parseInt(g, 16)); + return n; +} + +const pack = (c) => (c.length === 2 && c !== 'ZZ' && /^[A-Za-z]{2}$/.test(c) ? (c.toUpperCase().charCodeAt(0) << 8) | c.toUpperCase().charCodeAt(1) : 0); +const unpack = (n) => (n ? String.fromCharCode(n >> 8, n & 255) : ''); + +function sortedBy(n, cmp) { + for (let i = 1; i < n; i++) if (cmp(i - 1, i) > 0) return false; + return true; +} + +export function loadGeo(file) { + const none = { lookup: () => '', size: 0 }; + let text; + try { text = readFileSync(file, 'utf8'); } catch (e) { + console.warn(`Analytics: Länderdatei ${file} nicht lesbar (${e.code || e.message}); Länder bleiben leer.`); + return none; + } + // Obergrenze der Zeilen für die Vorab-Reservierung + let cap = 1; + for (let i = text.indexOf('\n'); i !== -1; i = text.indexOf('\n', i + 1)) cap++; + let n4 = 0; + let n6 = 0; + const s4 = new Uint32Array(cap); const e4 = new Uint32Array(cap); const c4 = new Uint16Array(cap); + const sh = new BigUint64Array(cap); const sl = new BigUint64Array(cap); + const eh = new BigUint64Array(cap); const el = new BigUint64Array(cap); const c6 = new Uint16Array(cap); + let pos = 0; + while (pos < text.length) { + let nl = text.indexOf('\n', pos); + if (nl === -1) nl = text.length; + const line = text.slice(pos, nl).replace(/["\r]/g, '').trim(); + pos = nl + 1; + if (!line) continue; + const p = line.split(','); + if (p.length < 3) continue; + const a = p[0].trim(); const b = p[1].trim(); const c = pack(p[2].trim()); + if (a.includes(':')) { + const x = parseV6(a); const y = parseV6(b); + if (x === null || y === null || y < x) continue; + sh[n6] = x >> 64n; sl[n6] = x & MASK64; eh[n6] = y >> 64n; el[n6] = y & MASK64; c6[n6++] = c; + } else { + const x = parseV4(a); const y = parseV4(b); + if (x < 0 || y < x) continue; + s4[n4] = x; e4[n4] = y; c4[n4++] = c; + } + } + text = null; + + // Falls die Datei nicht sortiert ist: Indexsortierung + let S4 = s4.slice(0, n4); let E4 = e4.slice(0, n4); let C4 = c4.slice(0, n4); + if (!sortedBy(n4, (i, j) => S4[i] - S4[j])) { + const idx = Array.from({ length: n4 }, (_, i) => i).sort((i, j) => S4[i] - S4[j]); + S4 = Uint32Array.from(idx, (i) => s4[i]); E4 = Uint32Array.from(idx, (i) => e4[i]); C4 = Uint16Array.from(idx, (i) => c4[i]); + } + let SH = sh.slice(0, n6); let SL = sl.slice(0, n6); let EH = eh.slice(0, n6); let EL = el.slice(0, n6); let C6 = c6.slice(0, n6); + const cmp6 = (i, j) => (SH[i] !== SH[j] ? (SH[i] < SH[j] ? -1 : 1) : (SL[i] === SL[j] ? 0 : (SL[i] < SL[j] ? -1 : 1))); + if (!sortedBy(n6, cmp6)) { + const idx = Array.from({ length: n6 }, (_, i) => i).sort(cmp6); + const pick = (A, T) => T.from(idx, (i) => A[i]); + const [a, b, c, d, e] = [pick(SH, BigUint64Array), pick(SL, BigUint64Array), pick(EH, BigUint64Array), pick(EL, BigUint64Array), pick(C6, Uint16Array)]; + SH = a; SL = b; EH = c; EL = d; C6 = e; + } + + const look4 = (v) => { + let lo = 0; let hi = n4 - 1; let best = -1; + while (lo <= hi) { + const mid = (lo + hi) >>> 1; + if (S4[mid] <= v) { best = mid; lo = mid + 1; } else hi = mid - 1; + } + return best >= 0 && v <= E4[best] ? unpack(C4[best]) : ''; + }; + const look6 = (h, l) => { + let lo = 0; let hi = n6 - 1; let best = -1; + while (lo <= hi) { + const mid = (lo + hi) >>> 1; + if (SH[mid] < h || (SH[mid] === h && SL[mid] <= l)) { best = mid; lo = mid + 1; } else hi = mid - 1; + } + if (best < 0) return ''; + const inside = h < EH[best] || (h === EH[best] && l <= EL[best]); + return inside ? unpack(C6[best]) : ''; + }; + + return { + size: n4 + n6, + lookup(ip) { + if (typeof ip !== 'string') return ''; + ip = ip.trim(); + if (!ip.includes(':')) { + const v = parseV4(ip); + return v < 0 ? '' : look4(v); + } + const v = parseV6(ip); + if (v === null) return ''; + const h = v >> 64n; const l = v & MASK64; + if (h === 0n && (l >> 32n) === 0xffffn) return look4(Number(l & 0xffffffffn)); // ::ffff:a.b.c.d + return look6(h, l); + }, + }; +} diff --git a/analytics/src/index.js b/analytics/src/index.js index 7f679dd..4f422ca 100644 --- a/analytics/src/index.js +++ b/analytics/src/index.js @@ -3,13 +3,14 @@ import { join } from 'node:path'; import { loadConfig } from './config.js'; import { openDb } from './db.js'; import { createSalt } from './salt.js'; +import { loadGeo } from './geo.js'; import { createPublicServer, createAdminServer } from './server.js'; -// Platzhalter bis Task 3 (loadGeo aus config.geoFile): kein Land. -const geo = { lookup: () => '' }; const config = loadConfig(process.env); mkdirSync(config.dataDir, { recursive: true }); const db = openDb(join(config.dataDir, 'analytics.db')); +const geo = loadGeo(config.geoFile || join(config.dataDir, 'geo.csv')); +if (geo.size) console.log(`Analytics: ${geo.size} Länder-Bereiche geladen`); const servers = [ ['öffentlich', createPublicServer({ config, db, geo, salt: createSalt() }), config.port], diff --git a/analytics/test/geo.test.js b/analytics/test/geo.test.js new file mode 100644 index 0000000..6aadd09 --- /dev/null +++ b/analytics/test/geo.test.js @@ -0,0 +1,110 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { loadGeo } from '../src/geo.js'; + +const CSV = [ + '1.0.0.0,1.0.0.255,AU', + '5.9.0.0,5.9.255.255,DE', + '5.10.0.0,5.10.0.9,ZZ', + '8.8.8.0,8.8.8.255,US', + '255.255.255.0,255.255.255.255,FR', + '2001:db8::,2001:db8:ffff:ffff:ffff:ffff:ffff:ffff,AT', + '2a01:4f8::,2a01:4f8:ffff:ffff:ffff:ffff:ffff:ffff,DE', + '2a02:1::1,2a02:1::ff,CH', + '', +].join('\n'); + +function fixture(content) { + const dir = mkdtempSync(join(tmpdir(), 'geo-')); + const f = join(dir, 'geo.csv'); + writeFileSync(f, content); + return f; +} + +test('IPv4: Treffer und Grenzen', () => { + const g = loadGeo(fixture(CSV)); + assert.equal(g.size, 8); + assert.equal(g.lookup('5.9.0.0'), 'DE'); + assert.equal(g.lookup('5.9.123.4'), 'DE'); + assert.equal(g.lookup('5.9.255.255'), 'DE'); + assert.equal(g.lookup('5.8.255.255'), ''); + assert.equal(g.lookup('5.10.0.0'), ''); + assert.equal(g.lookup('1.0.0.255'), 'AU'); + assert.equal(g.lookup('1.0.1.0'), ''); + assert.equal(g.lookup('0.0.0.1'), ''); + assert.equal(g.lookup('255.255.255.255'), 'FR'); + assert.equal(g.lookup('8.8.8.8'), 'US'); +}); + +test('IPv6: Treffer, Grenzen, Kurzschreibweise', () => { + const g = loadGeo(fixture(CSV)); + assert.equal(g.lookup('2a01:4f8::'), 'DE'); + assert.equal(g.lookup('2a01:4f8:1:2:3:4:5:6'), 'DE'); + assert.equal(g.lookup('2A01:04F8:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF'), 'DE'); + assert.equal(g.lookup('2a01:4f9::'), ''); + assert.equal(g.lookup('2a01:4f7:ffff:ffff:ffff:ffff:ffff:ffff'), ''); + assert.equal(g.lookup('2001:db8::1'), 'AT'); + assert.equal(g.lookup('2a02:1::1'), 'CH'); + assert.equal(g.lookup('2a02:1::ff'), 'CH'); + assert.equal(g.lookup('2a02:1::100'), ''); + assert.equal(g.lookup('2a02:1::'), ''); + assert.equal(g.lookup('::1'), ''); +}); + +test('IPv4-gemappte IPv6-Adressen zählen als IPv4', () => { + const g = loadGeo(fixture(CSV)); + assert.equal(g.lookup('::ffff:5.9.1.1'), 'DE'); + assert.equal(g.lookup('::FFFF:8.8.8.8'), 'US'); + assert.equal(g.lookup('::ffff:808:808'), 'US'); + assert.equal(g.lookup('::ffff:9.9.9.9'), ''); +}); + +test('ZZ, ungültige Eingaben, unsortierte Datei', () => { + const g = loadGeo(fixture(CSV)); + assert.equal(g.lookup('5.10.0.5'), ''); + for (const bad of ['', 'abc', '1.2.3', '256.1.1.1', '1.2.3.4.5', '2001:db8::zz', ':::', '1::2::3', null, undefined, 42]) { + assert.equal(g.lookup(bad), '', String(bad)); + } + const u = loadGeo(fixture('8.8.8.0,8.8.8.255,US\n1.0.0.0,1.0.0.255,AU\n2a01:4f8::,2a01:4f8::ff,DE\n2001:db8::,2001:db8::ff,AT\n')); + assert.equal(u.lookup('1.0.0.7'), 'AU'); + assert.equal(u.lookup('8.8.8.8'), 'US'); + assert.equal(u.lookup('2001:db8::7'), 'AT'); + assert.equal(u.lookup('2a01:4f8::7'), 'DE'); +}); + +test('fehlende Datei: alles leer, eine Warnung, kein Absturz', () => { + const warns = []; + const orig = console.warn; + console.warn = (...a) => warns.push(a.join(' ')); + try { + const g = loadGeo('/nicht/vorhanden/geo.csv'); + assert.equal(g.lookup('5.9.1.1'), ''); + assert.equal(g.lookup('2a01:4f8::1'), ''); + assert.equal(g.size, 0); + } finally { console.warn = orig; } + assert.equal(warns.length, 1); + assert.match(warns[0], /nicht lesbar/); +}); + +test('600k Zeilen: lädt zügig, Heap bleibt klein', () => { + const lines = []; + for (let i = 0; i < 400000; i++) lines.push(`${(i * 4096) >>> 24}.${((i * 4096) >>> 16) & 255}.${((i * 4096) >>> 8) & 255}.0,${(i * 4096) >>> 24}.${((i * 4096) >>> 16) & 255}.${((i * 4096 + 255) >>> 8) & 255}.255,${i % 7 ? 'DE' : 'AT'}`); + for (let i = 0; i < 200000; i++) lines.push(`2a00:${(i >> 8).toString(16)}:${(i & 255).toString(16)}::,2a00:${(i >> 8).toString(16)}:${(i & 255).toString(16)}:ffff:ffff:ffff:ffff:ffff,${i % 5 ? 'DE' : 'FR'}`); + const f = fixture(lines.join('\n')); + lines.length = 0; + globalThis.gc?.(); + const before = process.memoryUsage().heapUsed; const rss0 = process.memoryUsage().rss; + const t0 = Date.now(); + const g = loadGeo(f); + const ms = Date.now() - t0; + globalThis.gc?.(); + const grow = (process.memoryUsage().heapUsed - before) / 1048576; + console.log(`# geo 600k: ${ms} ms, Heap +${grow.toFixed(1)} MB, RSS +${((process.memoryUsage().rss - rss0) / 1048576).toFixed(0)} MB, Zeilen ${g.size}`); + assert.equal(g.size, 600000); + assert.equal(g.lookup('0.0.16.5'), 'DE'); + assert.equal(g.lookup('2a00:0:1::5'), 'DE'); + assert.ok(grow < 100, `Heap +${grow} MB`); +});