// Build-Aufträge: Einzel-Slot-Warteschlange, Klonen, Erkennung, Installation/Build im Container, atomare Veröffentlichung. import crypto from 'node:crypto'; import { spawn } from 'node:child_process'; import * as nodeFs from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, dirname, sep } from 'node:path'; import { buildRunArgs } from './docker.js'; import { detectFromDir } from './detect.js'; import { readMemAvailable } from './server.js'; import { injectAnalytics } from './analytics-inject.js'; const LOG_LINES = 300; const MAX_LINE = 2000; const KEEP_MS = 60 * 60e3; const MEM_RETRY_MS = 30e3; const DAY = 24 * 3600e3; const UID = 1000; // ---- Dateisystem-Helfer (überschreibbar für Tests) ---- async function chownR(p) { const st = await nodeFs.lstat(p); await nodeFs.lchown(p, UID, UID); if (st.isDirectory()) for (const e of await nodeFs.readdir(p)) await chownR(join(p, e)); } // Kopiert nur Verzeichnisse und reguläre Dateien (keine Symlinks, nie .git), Rechte 755/644 (R7). async function copyTree(fs, src, dst, { skipGit = false, skipNodeModules = false } = {}) { await fs.mkdir(dst, { recursive: true }); await fs.chmod(dst, 0o755); for (const e of await fs.readdir(src, { withFileTypes: true })) { if (skipGit && e.name === '.git') continue; if (skipNodeModules && e.name === 'node_modules') continue; const s = join(src, e.name); const d = join(dst, e.name); if (e.isDirectory()) await copyTree(fs, s, d); else if (e.isFile()) { await fs.copyFile(s, d); await fs.chmod(d, 0o644); } } } async function dirSize(fs, p) { const st = await fs.lstat(p); if (!st.isDirectory()) return st.size; let sum = 0; for (const e of await fs.readdir(p)) sum += await dirSize(fs, join(p, e)); return sum; } // ---- index.html-Rückfall (1:1 aus server/services/previewDeploy.js, Runner kann Portalcode nicht importieren) ---- function placeholderHtml(displayName) { const name = String(displayName || 'Diese Website') .replace(/&/g, '&') .replace(//g, '>'); return ` ${name} – wird vorbereitet

${name}

Diese Seite wird gerade vorbereitet oder aktualisiert.
Bitte in Kürze erneut laden.

`; } async function findIndexCandidate(fs, dir) { let entries = []; try { entries = await fs.readdir(dir, { withFileTypes: true }); } catch { return null; } const htmlFiles = entries .filter((e) => e.isFile() && /\.html?$/i.test(e.name) && !e.name.startsWith('.')) .map((e) => e.name) .filter((name) => name.toLowerCase() !== 'index.html'); if (!htmlFiles.length) return null; const preferred = ['index.htm', 'home.html', 'homepage.html', 'main.html', 'start.html', 'app.html']; for (const want of preferred) { const hit = htmlFiles.find((name) => name.toLowerCase() === want); if (hit) return hit; } if (htmlFiles.length === 1) return htmlFiles[0]; const sized = await Promise.all(htmlFiles.map(async (name) => { try { return { name, size: (await fs.stat(join(dir, name))).size }; } catch { return { name, size: 0 }; } })); sized.sort((a, b) => b.size - a.size); return sized[0].name; } // Stellt sicher, dass dir/index.html existiert: Kandidat (Kopie) oder Platzhalter; vorhandene index.html bleibt unberührt. async function ensureIndexHtml(fs, dir, displayName, onLine) { const indexPath = join(dir, 'index.html'); try { await fs.access(indexPath); return; } catch { /* keine index.html */ } const candidate = await findIndexCandidate(fs, dir); if (candidate) { await fs.copyFile(join(dir, candidate), indexPath); await fs.chmod(indexPath, 0o644); onLine?.(`keine index.html – "${candidate}" als Startseite übernommen`); return; } await fs.writeFile(indexPath, placeholderHtml(displayName), 'utf8'); await fs.chmod(indexPath, 0o644); onLine?.('keine HTML-Datei gefunden – Platzhalter-Startseite geschrieben'); } const defaultFs = { ...nodeFs, chownR, copyTree, dirSize: (p) => dirSize(nodeFs, p) }; // ---- git (R11) ---- const CLONE_MAX_MS = 5 * 60e3; export function createGit(config, { spawnFn = spawn } = {}) { return function git({ repo, branch, dir, onLine, timeoutMs = CLONE_MAX_MS }) { const b64 = Buffer.from(`x-access-token:${config.giteaToken}`).toString('base64'); const url = `${String(config.giteaUrl).replace(/\/+$/, '')}/${repo}.git`; const args = ['-c', 'http.lowSpeedLimit=1000', '-c', 'http.lowSpeedTime=60', 'clone', '--depth', '1', '--branch', branch, '--', url, dir]; const env = { PATH: process.env.PATH || '/usr/bin:/bin', GIT_TERMINAL_PROMPT: '0', GIT_CONFIG_NOSYSTEM: '1', HOME: tmpdir(), // Token nur über die Umgebung des git-Prozesses, nie in argv/.git/config (R11) GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'http.extraHeader', GIT_CONFIG_VALUE_0: `Authorization: Basic ${b64}` }; return new Promise((resolve, reject) => { const child = spawnFn('git', args, { env, stdio: ['ignore', 'pipe', 'pipe'] }); let timedOut = false; const timer = setTimeout(() => { timedOut = true; try { child.kill('SIGKILL'); } catch { /* beendet */ } reject(new Error('git clone: Zeitlimit überschritten')); }, Math.max(1000, Math.min(timeoutMs, CLONE_MAX_MS))); const feed = (s) => { let buf = ''; s.setEncoding('utf8'); s.on('data', (d) => { const parts = (buf + d).split(/\r?\n|\r/); buf = parts.pop(); parts.forEach((l) => l && onLine?.(l)); }); s.on('end', () => buf && onLine?.(buf)); }; feed(child.stdout); feed(child.stderr); child.on('error', (e) => { clearTimeout(timer); reject(e); }); child.on('close', (code) => { clearTimeout(timer); if (!timedOut) { if (code === 0) resolve(); else reject(new Error(`git clone fehlgeschlagen (Exit-Code ${code})`)); } }); }); }; } const withCorepack = (pm, cmd) => (pm === 'pnpm' || pm === 'yarn' ? ['corepack', ...cmd] : cmd); // R9 const imageFor = (pm) => (pm === 'bun' ? 'oven/bun:1' : 'node:20'); const cacheName = (repo) => repo.replace('/', '__'); export function createJobs({ config, docker, fs: fsOver = {}, git, mem = readMemAvailable, now = Date.now, setTimer = (fn, ms) => setTimeout(fn, ms).unref?.() ?? 0 }) { const fs = { ...defaultFs, ...fsOver }; const gitFn = git || createGit(config); const runs = new Map(); const queue = []; let active = false; let retry = false; const activeCaches = new Map(); // Cache-Name ⇒ Anzahl Halter // ---- Warteschlange mit genau einem aktiven Slot und Speicherwächter ---- function pump() { if (active || queue.length === 0) return; let free = 0; try { free = mem(); } catch { /* Fehler = zu wenig Speicher */ } if (!(free >= config.minFreeBytes)) { if (!retry) { retry = true; setTimer(() => { retry = false; pump(); }, MEM_RETRY_MS); } return; } active = true; const task = queue.shift(); Promise.resolve().then(task).finally(() => { active = false; pump(); }); } const enqueue = (fn) => new Promise((resolve, reject) => { queue.push(() => fn().then(resolve, reject)); pump(); }); function purge() { const t = now(); for (const [id, r] of runs) if (r.finishedAt && t - r.finishedAt > KEEP_MS) runs.delete(id); } // ---- Log: Ringpuffer, Token/Variablenwerte maskiert ---- function masker(job) { const secrets = []; const tok = config.giteaToken; if (tok) secrets.push(tok, Buffer.from(`x-access-token:${tok}`).toString('base64'), Buffer.from(`${tok}:`).toString('base64')); for (const v of Object.values(job?.env || {})) if (typeof v === 'string' && v.length >= 4) secrets.push(v); secrets.sort((a, b) => b.length - a.length); return (line) => { let s = String(line).slice(0, MAX_LINE); for (const x of secrets) s = s.split(x).join('•••'); return s.replace(/(Authorization:\s*)(Basic|Bearer)\s+\S+/gi, '$1$2 •••').replace(/(https?:\/\/)[^/\s@]+@/g, '$1•••@'); }; } const makeLog = (log, mask) => (line) => { log.push(mask(line)); if (log.length > LOG_LINES) log.splice(0, log.length - LOG_LINES); }; // ---- Container-Schritt ---- async function containerStep({ job, id, step, cmd, pm, workDir, cacheDir, env, onLine, deadline }) { await fs.mkdir(cacheDir, { recursive: true }); const t = new Date(now()); await fs.utimes(cacheDir, t, t).catch(() => {}); // Alter = letzte Nutzung await fs.chownR(workDir); await fs.chownR(cacheDir); // R7 const remaining = Math.max(1000, deadline - now()); const args = buildRunArgs({ image: imageFor(pm), name: `preview-build-${id}-${step}`, workDir, cacheDir, workRoot: config.workRoot, cacheRoot: config.cacheRoot, cmd: withCorepack(pm, cmd), memory: '2g' }); const r = await docker.run(args, { env, timeoutMs: remaining, onLine: (l) => onLine(l) }); if (r && r.code === 137) onLine('Build am Speicherlimit abgebrochen (Exit 137)'); return r; } const failText = (what, code, limit) => (code === 137 ? 'Build am Speicherlimit abgebrochen (Exit 137)' : code === 124 ? `${what}: Zeitlimit von ${Math.round(limit / 60e3)} Minuten überschritten` : `${what} fehlgeschlagen (Exit-Code ${code})`); // Klonen + Erkennung + Installation (R1); Aufrufer hält den Slot. async function prepare(job, { onLine, workDir, id = crypto.randomUUID(), deadline = now() + config.buildTimeoutMs }) { const cacheDir = join(config.cacheRoot, cacheName(job.repo)); await fs.rm(workDir, { recursive: true, force: true }); await fs.mkdir(dirname(workDir), { recursive: true }); await gitFn({ repo: job.repo, branch: job.branch, dir: workDir, onLine, timeoutMs: Math.max(1000, deadline - now()) }); const detected = detectFromDir(workDir, { mode: job.mode, index: job.index }); if (detected.error || detected.kind === 'static' || !detected.install?.length) return { workDir, detected }; let r = await containerStep({ job, id, step: 'install', cmd: detected.install, pm: detected.pm, workDir, cacheDir, onLine, deadline }); // I8: `npm ci` scheitert bei nicht passender Lock-Datei; wie der bisherige Build-Weg auf `npm install` ausweichen. if (r.code !== 0 && detected.pm === 'npm' && detected.install[1] === 'ci') { onLine('npm ci fehlgeschlagen – Wiederholung mit npm install'); r = await containerStep({ job, id, step: 'install2', cmd: ['npm', 'install', '--include=dev', '--no-audit', '--no-fund'], pm: detected.pm, workDir, cacheDir, onLine, deadline }); } if (r.code !== 0) return { workDir, detected, error: failText('Installation', r.code, config.buildTimeoutMs) }; return { workDir, detected }; } // Cache vor pruneCache schützen; release() gibt ihn frei (idempotent). Zählend, mehrere Halter je Repo möglich. function retainCache(repo) { const n = cacheName(repo); activeCaches.set(n, (activeCaches.get(n) || 0) + 1); let done = false; return () => { if (done) return; done = true; const c = (activeCaches.get(n) || 1) - 1; if (c <= 0) activeCaches.delete(n); else activeCaches.set(n, c); }; } async function holdCache(repo, fn) { const release = retainCache(repo); try { return await fn(); } finally { release(); } } // ---- Atomare Veröffentlichung (R11) ---- async function publish(job, srcDir, { skipGit, onLine, skipNodeModules = false }) { const target = join(config.previewsRoot, job.sub); const fresh = `${target}.new`; const old = `${target}.old`; await fs.rm(fresh, { recursive: true, force: true }); // Abgebrochener früherer Tausch: fehlt , ist .old die letzte gute Kopie – zurückholen. const exists = (p) => fs.lstat(p).then(() => true, () => false); if (!(await exists(target)) && (await exists(old))) await fs.rename(old, target); await fs.rm(old, { recursive: true, force: true }); let hadOld = true; try { await fs.mkdir(config.previewsRoot, { recursive: true }); await fs.copyTree(fs, srcDir, fresh, { skipGit, skipNodeModules }); await ensureIndexHtml(fs, fresh, job.displayName || job.sub, onLine); // Analyse (optional): auf dem .new-Ordner vor dem Tausch; Fehler brechen die Veröffentlichung ab, die alte Preview bleibt (V4). if (job.analytics) await injectAnalytics(fresh, job.analytics, { baseUrl: config.analyticsPublicBase, onLine, fs }); try { await fs.rename(target, old); } catch (e) { if (e.code !== 'ENOENT') throw e; hadOld = false; } try { await fs.rename(fresh, target); } catch (e) { if (hadOld) await fs.rename(old, target).catch((e2) => onLine?.(`Rollback fehlgeschlagen: ${e2.message}`)); throw e; } } catch (e) { await fs.rm(fresh, { recursive: true, force: true }).catch(() => {}); throw e; } // Ab hier ist der Preview live; Aufräumen darf den Lauf nicht scheitern lassen. await fs.rm(old, { recursive: true, force: true }).catch((e) => onLine?.(`Aufräumen fehlgeschlagen: ${e.message}`)); } async function findOutput(workDir, dirs) { const realWork = await fs.realpath(workDir); for (const d of dirs) { const p = join(workDir, d); try { const st = await fs.lstat(p); if (!st.isDirectory()) continue; const real = await fs.realpath(p); if (real === realWork || !real.startsWith(realWork + sep)) continue; await fs.access(join(p, 'index.html')); // wie bisher: nur Ausgabeordner mit index.html zählen return p; } catch { /* nicht vorhanden / ohne index.html */ } } return null; } // Webroot der statischen Seite: index-Unterordner (wie bisher), sonst Repo-Wurzel; fehlender/unsicherer Ordner ⇒ Wurzel. async function webrootOf(workDir, sub) { if (!sub) return workDir; try { const p = join(workDir, sub); if (!(await fs.lstat(p)).isDirectory()) return workDir; const real = await fs.realpath(p); const realWork = await fs.realpath(workDir); if (real === realWork || !real.startsWith(realWork + sep)) return workDir; return p; } catch { return workDir; } } // Erstes Deploy (noch kein -Ordner): Quellstand statisch veröffentlichen statt 403/leer (Verhalten des bisherigen Pfads). async function firstDeployFallback(job, workDir, d, problem, onLine) { if (job.mode === 'build') return false; const target = join(config.previewsRoot, job.sub); if (await fs.lstat(target).then(() => true, () => false)) return false; onLine(`${problem} – erstes Deploy, Quellstand wird statisch veröffentlicht (Hinweis: Build fehlgeschlagen)`); await publish(job, await webrootOf(workDir, job.index || d.index || ''), { skipGit: true, skipNodeModules: true, onLine }); return true; } // ---- Ablauf eines Build-Laufs ---- const execute = (run, job) => holdCache(job.repo, () => executeRun(run, job)); async function executeRun(run, job) { const onLine = makeLog(run.log, masker(job)); const workDir = join(config.workRoot, 'build', run.runId); const deadline = now() + config.buildTimeoutMs; run.state = 'running'; let final; try { const prep = await prepare(job, { onLine, workDir, id: run.runId, deadline }); const d = prep.detected; run.detected = d; if (d.error) throw new Error(d.error); // Build-/Installationsproblem: erstes Deploy ⇒ Quellstand statisch (wie bisher), sonst Fehler (alte Preview bleibt) const fallbackOr = async (problem) => { if (!(await firstDeployFallback(job, workDir, d, problem, onLine))) throw new Error(problem); run.result = 'static'; run.hint = 'Build fehlgeschlagen – Quellstand statisch veröffentlicht'; }; if (prep.error) { await fallbackOr(prep.error); final = 'done'; } else if (d.kind === 'static') { if (d.hint) onLine(d.hint); await publish(job, await webrootOf(workDir, d.index || job.index || ''), { skipGit: true, onLine }); run.result = 'static'; } else if (d.kind === 'dev') { run.result = 'dev'; } else { let problem = null; const cacheDir = join(config.cacheRoot, cacheName(job.repo)); const r = await containerStep({ job, id: run.runId, step: 'build', cmd: d.buildCmd, pm: d.pm, workDir, cacheDir, env: job.env, onLine, deadline }); if (r.code !== 0) problem = failText('Build', r.code, config.buildTimeoutMs); let out = null; if (!problem) { out = await findOutput(workDir, d.outputDirs); if (!out) problem = `Kein Ausgabeordner gefunden (gesucht: ${d.outputDirs.join(', ')})`; } if (problem) { const hasFolder = await fs.lstat(join(config.previewsRoot, job.sub)).then(() => true, () => false); // Dev-Rueckfall nur beim ersten Deploy; mit bestehender Preview bleibt sie (V4) und das Routing auf nginx if (job.mode === 'auto' && d.devCmd && !d.explicitType && !hasFolder) { onLine(`${problem} – Dev-Server wird verwendet`); run.result = 'dev'; } else await fallbackOr(problem); } else { await publish(job, out, { skipGit: false, onLine }); run.result = 'build'; } } final = 'done'; } catch (e) { final = 'failed'; run.result = undefined; run.error = masker(job)(e?.message || 'Unbekannter Fehler'); onLine(run.error); } // Endzustand erst nach dem Aufräumen veröffentlichen: wer 'done'/'failed' sieht, sieht ein bereinigtes workDir und finishedAt. await fs.rm(workDir, { recursive: true, force: true }).catch(() => {}); run.finishedAt = now(); run.state = final; } function submit(job) { purge(); const runId = crypto.randomUUID(); const run = { runId, state: 'queued', detected: null, sha: job.sha, log: [], error: null, result: null, finishedAt: 0 }; runs.set(runId, run); enqueue(() => execute(run, job)); return runId; } function get(runId) { purge(); const r = runs.get(runId); if (!r) return null; const { finishedAt, ...pub } = r; // eslint-disable-line no-unused-vars return { ...pub, log: [...r.log] }; } // Für Dev (Task 5): gleicher Slot, Klon + Installation in workDir. function prepareWorkspace(job, { onLine = () => {}, workDir } = {}) { const mask = masker(job); const log = (l) => onLine(mask(l)); return enqueue(() => holdCache(job.repo, async () => { const r = await prepare(job, { onLine: log, workDir }); if (r.error) throw new Error(r.error); return { workDir: r.workDir, detected: r.detected }; })); } async function pruneCache() { const root = config.cacheRoot; const maxAge = config.cacheMaxAgeMs ?? 14 * DAY; const maxBytes = config.cacheMaxBytes ?? 5 * 1024 ** 3; let names = []; try { names = await fs.readdir(root); } catch { return; } const items = []; for (const n of names) { if (activeCaches.has(n)) continue; try { const p = join(root, n); const st = await fs.lstat(p); items.push({ p, mtime: st.mtimeMs, size: await fs.dirSize(p) }); } catch { /* weg */ } } let kept = []; for (const it of items) { if (now() - it.mtime > maxAge) await fs.rm(it.p, { recursive: true, force: true }); else kept.push(it); } kept.sort((a, b) => a.mtime - b.mtime); let total = kept.reduce((s, i) => s + i.size, 0); for (const it of kept) { if (total <= maxBytes) break; await fs.rm(it.p, { recursive: true, force: true }); total -= it.size; } } return { submit, get, queueLength: () => queue.length + (active ? 1 : 0), prepareWorkspace, retainCache, pruneCache }; }