// Build-Aufträge: Einzel-Slot-Warteschlange, Klonen, Erkennung, Installation/Build im Container, atomare Veröffentlichung.
import crypto from 'node:crypto';
import { spawn } from 'node:child_process';
import * as nodeFs from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, dirname, sep } from 'node:path';
import { buildRunArgs } from './docker.js';
import { detectFromDir } from './detect.js';
import { readMemAvailable } from './server.js';
import { injectAnalytics, injectZeiten } from './analytics-inject.js';
const LOG_LINES = 300;
const MAX_LINE = 2000;
const KEEP_MS = 60 * 60e3;
const MEM_RETRY_MS = 30e3;
const DAY = 24 * 3600e3;
const UID = 1000;
// ---- Dateisystem-Helfer (überschreibbar für Tests) ----
async function chownR(p) {
const st = await nodeFs.lstat(p);
await nodeFs.lchown(p, UID, UID);
if (st.isDirectory()) for (const e of await nodeFs.readdir(p)) await chownR(join(p, e));
}
// Kopiert nur Verzeichnisse und reguläre Dateien (keine Symlinks, nie .git), Rechte 755/644 (R7).
async function copyTree(fs, src, dst, { skipGit = false, skipNodeModules = false } = {}) {
await fs.mkdir(dst, { recursive: true });
await fs.chmod(dst, 0o755);
for (const e of await fs.readdir(src, { withFileTypes: true })) {
if (skipGit && e.name === '.git') continue;
if (skipNodeModules && e.name === 'node_modules') continue;
const s = join(src, e.name);
const d = join(dst, e.name);
if (e.isDirectory()) await copyTree(fs, s, d);
else if (e.isFile()) { await fs.copyFile(s, d); await fs.chmod(d, 0o644); }
}
}
async function dirSize(fs, p) {
const st = await fs.lstat(p);
if (!st.isDirectory()) return st.size;
let sum = 0;
for (const e of await fs.readdir(p)) sum += await dirSize(fs, join(p, e));
return sum;
}
// ---- index.html-Rückfall (1:1 aus server/services/previewDeploy.js, Runner kann Portalcode nicht importieren) ----
function placeholderHtml(displayName) {
const name = String(displayName || 'Diese Website')
.replace(/&/g, '&')
.replace(//g, '>');
return `
${name} – wird vorbereitet
${name}
Diese Seite wird gerade vorbereitet oder aktualisiert.
Bitte in Kürze erneut laden.
`;
}
async function findIndexCandidate(fs, dir) {
let entries = [];
try { entries = await fs.readdir(dir, { withFileTypes: true }); } catch { return null; }
const htmlFiles = entries
.filter((e) => e.isFile() && /\.html?$/i.test(e.name) && !e.name.startsWith('.'))
.map((e) => e.name)
.filter((name) => name.toLowerCase() !== 'index.html');
if (!htmlFiles.length) return null;
const preferred = ['index.htm', 'home.html', 'homepage.html', 'main.html', 'start.html', 'app.html'];
for (const want of preferred) {
const hit = htmlFiles.find((name) => name.toLowerCase() === want);
if (hit) return hit;
}
if (htmlFiles.length === 1) return htmlFiles[0];
const sized = await Promise.all(htmlFiles.map(async (name) => {
try { return { name, size: (await fs.stat(join(dir, name))).size }; } catch { return { name, size: 0 }; }
}));
sized.sort((a, b) => b.size - a.size);
return sized[0].name;
}
// Stellt sicher, dass dir/index.html existiert: Kandidat (Kopie) oder Platzhalter; vorhandene index.html bleibt unberührt.
async function ensureIndexHtml(fs, dir, displayName, onLine) {
const indexPath = join(dir, 'index.html');
try { await fs.access(indexPath); return; } catch { /* keine index.html */ }
const candidate = await findIndexCandidate(fs, dir);
if (candidate) {
await fs.copyFile(join(dir, candidate), indexPath);
await fs.chmod(indexPath, 0o644);
onLine?.(`keine index.html – "${candidate}" als Startseite übernommen`);
return;
}
await fs.writeFile(indexPath, placeholderHtml(displayName), 'utf8');
await fs.chmod(indexPath, 0o644);
onLine?.('keine HTML-Datei gefunden – Platzhalter-Startseite geschrieben');
}
const defaultFs = { ...nodeFs, chownR, copyTree, dirSize: (p) => dirSize(nodeFs, p) };
// ---- git (R11) ----
const CLONE_MAX_MS = 5 * 60e3;
export function createGit(config, { spawnFn = spawn } = {}) {
return function git({ repo, branch, dir, onLine, timeoutMs = CLONE_MAX_MS }) {
const b64 = Buffer.from(`x-access-token:${config.giteaToken}`).toString('base64');
const url = `${String(config.giteaUrl).replace(/\/+$/, '')}/${repo}.git`;
const args = ['-c', 'http.lowSpeedLimit=1000', '-c', 'http.lowSpeedTime=60', 'clone', '--depth', '1', '--branch', branch, '--', url, dir];
const env = { PATH: process.env.PATH || '/usr/bin:/bin', GIT_TERMINAL_PROMPT: '0', GIT_CONFIG_NOSYSTEM: '1', HOME: tmpdir(),
// Token nur über die Umgebung des git-Prozesses, nie in argv/.git/config (R11)
GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'http.extraHeader', GIT_CONFIG_VALUE_0: `Authorization: Basic ${b64}` };
return new Promise((resolve, reject) => {
const child = spawnFn('git', args, { env, stdio: ['ignore', 'pipe', 'pipe'] });
let timedOut = false;
const timer = setTimeout(() => { timedOut = true; try { child.kill('SIGKILL'); } catch { /* beendet */ } reject(new Error('git clone: Zeitlimit überschritten')); }, Math.max(1000, Math.min(timeoutMs, CLONE_MAX_MS)));
const feed = (s) => { let buf = ''; s.setEncoding('utf8'); s.on('data', (d) => { const parts = (buf + d).split(/\r?\n|\r/); buf = parts.pop(); parts.forEach((l) => l && onLine?.(l)); }); s.on('end', () => buf && onLine?.(buf)); };
feed(child.stdout); feed(child.stderr);
child.on('error', (e) => { clearTimeout(timer); reject(e); });
child.on('close', (code) => { clearTimeout(timer); if (!timedOut) { if (code === 0) resolve(); else reject(new Error(`git clone fehlgeschlagen (Exit-Code ${code})`)); } });
});
};
}
const withCorepack = (pm, cmd) => (pm === 'pnpm' || pm === 'yarn' ? ['corepack', ...cmd] : cmd); // R9
const imageFor = (pm) => (pm === 'bun' ? 'oven/bun:1' : 'node:20');
const cacheName = (repo) => repo.replace('/', '__');
export function createJobs({ config, docker, fs: fsOver = {}, git, mem = readMemAvailable, now = Date.now, setTimer = (fn, ms) => setTimeout(fn, ms).unref?.() ?? 0 }) {
const fs = { ...defaultFs, ...fsOver };
const gitFn = git || createGit(config);
const runs = new Map();
const queue = [];
let active = false;
let retry = false;
const activeCaches = new Map(); // Cache-Name ⇒ Anzahl Halter
// ---- Warteschlange mit genau einem aktiven Slot und Speicherwächter ----
function pump() {
if (active || queue.length === 0) return;
let free = 0;
try { free = mem(); } catch { /* Fehler = zu wenig Speicher */ }
if (!(free >= config.minFreeBytes)) {
if (!retry) { retry = true; setTimer(() => { retry = false; pump(); }, MEM_RETRY_MS); }
return;
}
active = true;
const task = queue.shift();
Promise.resolve().then(task).finally(() => { active = false; pump(); });
}
const enqueue = (fn) => new Promise((resolve, reject) => {
queue.push(() => fn().then(resolve, reject));
pump();
});
function purge() {
const t = now();
for (const [id, r] of runs) if (r.finishedAt && t - r.finishedAt > KEEP_MS) runs.delete(id);
}
// ---- Log: Ringpuffer, Token/Variablenwerte maskiert ----
function masker(job) {
const secrets = [];
const tok = config.giteaToken;
if (tok) secrets.push(tok, Buffer.from(`x-access-token:${tok}`).toString('base64'), Buffer.from(`${tok}:`).toString('base64'));
for (const v of Object.values(job?.env || {})) if (typeof v === 'string' && v.length >= 4) secrets.push(v);
secrets.sort((a, b) => b.length - a.length);
return (line) => {
let s = String(line).slice(0, MAX_LINE);
for (const x of secrets) s = s.split(x).join('•••');
return s.replace(/(Authorization:\s*)(Basic|Bearer)\s+\S+/gi, '$1$2 •••').replace(/(https?:\/\/)[^/\s@]+@/g, '$1•••@');
};
}
const makeLog = (log, mask) => (line) => { log.push(mask(line)); if (log.length > LOG_LINES) log.splice(0, log.length - LOG_LINES); };
// ---- Container-Schritt ----
async function containerStep({ job, id, step, cmd, pm, workDir, cacheDir, env, onLine, deadline }) {
await fs.mkdir(cacheDir, { recursive: true });
const t = new Date(now());
await fs.utimes(cacheDir, t, t).catch(() => {}); // Alter = letzte Nutzung
await fs.chownR(workDir);
await fs.chownR(cacheDir); // R7
const remaining = Math.max(1000, deadline - now());
const args = buildRunArgs({ image: imageFor(pm), name: `preview-build-${id}-${step}`, workDir, cacheDir, workRoot: config.workRoot, cacheRoot: config.cacheRoot, cmd: withCorepack(pm, cmd), memory: '2g' });
const r = await docker.run(args, { env, timeoutMs: remaining, onLine: (l) => onLine(l) });
if (r && r.code === 137) onLine('Build am Speicherlimit abgebrochen (Exit 137)');
return r;
}
const failText = (what, code, limit) => (code === 137 ? 'Build am Speicherlimit abgebrochen (Exit 137)' : code === 124 ? `${what}: Zeitlimit von ${Math.round(limit / 60e3)} Minuten überschritten` : `${what} fehlgeschlagen (Exit-Code ${code})`);
// Klonen + Erkennung + Installation (R1); Aufrufer hält den Slot.
async function prepare(job, { onLine, workDir, id = crypto.randomUUID(), deadline = now() + config.buildTimeoutMs }) {
const cacheDir = join(config.cacheRoot, cacheName(job.repo));
await fs.rm(workDir, { recursive: true, force: true });
await fs.mkdir(dirname(workDir), { recursive: true });
await gitFn({ repo: job.repo, branch: job.branch, dir: workDir, onLine, timeoutMs: Math.max(1000, deadline - now()) });
const detected = detectFromDir(workDir, { mode: job.mode, index: job.index });
if (detected.error || detected.kind === 'static' || !detected.install?.length) return { workDir, detected };
let r = await containerStep({ job, id, step: 'install', cmd: detected.install, pm: detected.pm, workDir, cacheDir, onLine, deadline });
// I8: `npm ci` scheitert bei nicht passender Lock-Datei; wie der bisherige Build-Weg auf `npm install` ausweichen.
if (r.code !== 0 && detected.pm === 'npm' && detected.install[1] === 'ci') {
onLine('npm ci fehlgeschlagen – Wiederholung mit npm install');
r = await containerStep({ job, id, step: 'install2', cmd: ['npm', 'install', '--include=dev', '--no-audit', '--no-fund'], pm: detected.pm, workDir, cacheDir, onLine, deadline });
}
if (r.code !== 0) return { workDir, detected, error: failText('Installation', r.code, config.buildTimeoutMs) };
return { workDir, detected };
}
// Cache vor pruneCache schützen; release() gibt ihn frei (idempotent). Zählend, mehrere Halter je Repo möglich.
function retainCache(repo) {
const n = cacheName(repo);
activeCaches.set(n, (activeCaches.get(n) || 0) + 1);
let done = false;
return () => {
if (done) return;
done = true;
const c = (activeCaches.get(n) || 1) - 1;
if (c <= 0) activeCaches.delete(n); else activeCaches.set(n, c);
};
}
async function holdCache(repo, fn) {
const release = retainCache(repo);
try { return await fn(); } finally { release(); }
}
// ---- Atomare Veröffentlichung (R11) ----
async function publish(job, srcDir, { skipGit, onLine, skipNodeModules = false }) {
const target = join(config.previewsRoot, job.sub);
const fresh = `${target}.new`;
const old = `${target}.old`;
await fs.rm(fresh, { recursive: true, force: true });
// Abgebrochener früherer Tausch: fehlt , ist .old die letzte gute Kopie – zurückholen.
const exists = (p) => fs.lstat(p).then(() => true, () => false);
if (!(await exists(target)) && (await exists(old))) await fs.rename(old, target);
await fs.rm(old, { recursive: true, force: true });
let hadOld = true;
try {
await fs.mkdir(config.previewsRoot, { recursive: true });
await fs.copyTree(fs, srcDir, fresh, { skipGit, skipNodeModules });
await ensureIndexHtml(fs, fresh, job.displayName || job.sub, onLine);
// Analyse (optional): auf dem .new-Ordner vor dem Tausch; Fehler brechen die Veröffentlichung ab, die alte Preview bleibt (V4).
if (job.analytics) await injectAnalytics(fresh, job.analytics, { baseUrl: config.analyticsPublicBase, onLine, fs });
// Öffnungszeiten live: gleiche Runde, gleiche Fehlerfolge (Job failed, alte Preview bleibt). Nie im Dev-Pfad.
if (job.zeiten) await injectZeiten(fresh, job.zeiten, { baseUrl: config.zeitenPublicBase, onLine, fs });
try { await fs.rename(target, old); } catch (e) { if (e.code !== 'ENOENT') throw e; hadOld = false; }
try { await fs.rename(fresh, target); } catch (e) {
if (hadOld) await fs.rename(old, target).catch((e2) => onLine?.(`Rollback fehlgeschlagen: ${e2.message}`));
throw e;
}
} catch (e) {
await fs.rm(fresh, { recursive: true, force: true }).catch(() => {});
throw e;
}
// Ab hier ist der Preview live; Aufräumen darf den Lauf nicht scheitern lassen.
await fs.rm(old, { recursive: true, force: true }).catch((e) => onLine?.(`Aufräumen fehlgeschlagen: ${e.message}`));
}
async function findOutput(workDir, dirs) {
const realWork = await fs.realpath(workDir);
for (const d of dirs) {
const p = join(workDir, d);
try {
const st = await fs.lstat(p);
if (!st.isDirectory()) continue;
const real = await fs.realpath(p);
if (real === realWork || !real.startsWith(realWork + sep)) continue;
await fs.access(join(p, 'index.html')); // wie bisher: nur Ausgabeordner mit index.html zählen
return p;
} catch { /* nicht vorhanden / ohne index.html */ }
}
return null;
}
// Webroot der statischen Seite: index-Unterordner (wie bisher), sonst Repo-Wurzel; fehlender/unsicherer Ordner ⇒ Wurzel.
async function webrootOf(workDir, sub) {
if (!sub) return workDir;
try {
const p = join(workDir, sub);
if (!(await fs.lstat(p)).isDirectory()) return workDir;
const real = await fs.realpath(p);
const realWork = await fs.realpath(workDir);
if (real === realWork || !real.startsWith(realWork + sep)) return workDir;
return p;
} catch { return workDir; }
}
// Erstes Deploy (noch kein -Ordner): Quellstand statisch veröffentlichen statt 403/leer (Verhalten des bisherigen Pfads).
async function firstDeployFallback(job, workDir, d, problem, onLine) {
if (job.mode === 'build') return false;
const target = join(config.previewsRoot, job.sub);
if (await fs.lstat(target).then(() => true, () => false)) return false;
onLine(`${problem} – erstes Deploy, Quellstand wird statisch veröffentlicht (Hinweis: Build fehlgeschlagen)`);
await publish(job, await webrootOf(workDir, job.index || d.index || ''), { skipGit: true, skipNodeModules: true, onLine });
return true;
}
// ---- Ablauf eines Build-Laufs ----
const execute = (run, job) => holdCache(job.repo, () => executeRun(run, job));
async function executeRun(run, job) {
const onLine = makeLog(run.log, masker(job));
const workDir = join(config.workRoot, 'build', run.runId);
const deadline = now() + config.buildTimeoutMs;
run.state = 'running';
let final;
try {
const prep = await prepare(job, { onLine, workDir, id: run.runId, deadline });
const d = prep.detected;
run.detected = d;
if (d.error) throw new Error(d.error);
// Build-/Installationsproblem: erstes Deploy ⇒ Quellstand statisch (wie bisher), sonst Fehler (alte Preview bleibt)
const fallbackOr = async (problem) => {
if (!(await firstDeployFallback(job, workDir, d, problem, onLine))) throw new Error(problem);
run.result = 'static';
run.hint = 'Build fehlgeschlagen – Quellstand statisch veröffentlicht';
};
if (prep.error) { await fallbackOr(prep.error); final = 'done'; }
else
if (d.kind === 'static') {
if (d.hint) onLine(d.hint);
await publish(job, await webrootOf(workDir, d.index || job.index || ''), { skipGit: true, onLine });
run.result = 'static';
} else if (d.kind === 'dev') {
run.result = 'dev';
} else {
let problem = null;
const cacheDir = join(config.cacheRoot, cacheName(job.repo));
const r = await containerStep({ job, id: run.runId, step: 'build', cmd: d.buildCmd, pm: d.pm, workDir, cacheDir, env: job.env, onLine, deadline });
if (r.code !== 0) problem = failText('Build', r.code, config.buildTimeoutMs);
let out = null;
if (!problem) {
out = await findOutput(workDir, d.outputDirs);
if (!out) problem = `Kein Ausgabeordner gefunden (gesucht: ${d.outputDirs.join(', ')})`;
}
if (problem) {
const hasFolder = await fs.lstat(join(config.previewsRoot, job.sub)).then(() => true, () => false);
// Dev-Rueckfall nur beim ersten Deploy; mit bestehender Preview bleibt sie (V4) und das Routing auf nginx
if (job.mode === 'auto' && d.devCmd && !d.explicitType && !hasFolder) { onLine(`${problem} – Dev-Server wird verwendet`); run.result = 'dev'; }
else await fallbackOr(problem);
} else {
await publish(job, out, { skipGit: false, onLine });
run.result = 'build';
}
}
final = 'done';
} catch (e) {
final = 'failed';
run.result = undefined;
run.error = masker(job)(e?.message || 'Unbekannter Fehler');
onLine(run.error);
}
// Endzustand erst nach dem Aufräumen veröffentlichen: wer 'done'/'failed' sieht, sieht ein bereinigtes workDir und finishedAt.
await fs.rm(workDir, { recursive: true, force: true }).catch(() => {});
run.finishedAt = now();
run.state = final;
}
function submit(job) {
purge();
const runId = crypto.randomUUID();
const run = { runId, state: 'queued', detected: null, sha: job.sha, log: [], error: null, result: null, finishedAt: 0 };
runs.set(runId, run);
enqueue(() => execute(run, job));
return runId;
}
function get(runId) {
purge();
const r = runs.get(runId);
if (!r) return null;
const { finishedAt, ...pub } = r; // eslint-disable-line no-unused-vars
return { ...pub, log: [...r.log] };
}
// Für Dev (Task 5): gleicher Slot, Klon + Installation in workDir.
function prepareWorkspace(job, { onLine = () => {}, workDir } = {}) {
const mask = masker(job);
const log = (l) => onLine(mask(l));
return enqueue(() => holdCache(job.repo, async () => {
const r = await prepare(job, { onLine: log, workDir });
if (r.error) throw new Error(r.error);
return { workDir: r.workDir, detected: r.detected };
}));
}
async function pruneCache() {
const root = config.cacheRoot;
const maxAge = config.cacheMaxAgeMs ?? 14 * DAY;
const maxBytes = config.cacheMaxBytes ?? 5 * 1024 ** 3;
let names = [];
try { names = await fs.readdir(root); } catch { return; }
const items = [];
for (const n of names) {
if (activeCaches.has(n)) continue;
try { const p = join(root, n); const st = await fs.lstat(p); items.push({ p, mtime: st.mtimeMs, size: await fs.dirSize(p) }); } catch { /* weg */ }
}
let kept = [];
for (const it of items) {
if (now() - it.mtime > maxAge) await fs.rm(it.p, { recursive: true, force: true }); else kept.push(it);
}
kept.sort((a, b) => a.mtime - b.mtime);
let total = kept.reduce((s, i) => s + i.size, 0);
for (const it of kept) {
if (total <= maxBytes) break;
await fs.rm(it.p, { recursive: true, force: true });
total -= it.size;
}
}
return { submit, get, queueLength: () => queue.length + (active ? 1 : 0), prepareWorkspace, retainCache, pruneCache };
}