From 4fffd8b2180874796e1f0636c8bef7d98736f6b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 19:59:29 +0000 Subject: [PATCH] Runner: Build-Auftraege Review-Korrekturen Co-Authored-By: Claude Sonnet 5.5 --- runner/src/jobs.js | 79 ++++++++++++++++++++++++---------------- runner/test/jobs.test.js | 72 +++++++++++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 32 deletions(-) diff --git a/runner/src/jobs.js b/runner/src/jobs.js index 7f07e44..75999d0 100644 --- a/runner/src/jobs.js +++ b/runner/src/jobs.js @@ -46,18 +46,23 @@ async function dirSize(fs, p) { const defaultFs = { ...nodeFs, chownR, copyTree, dirSize: (p) => dirSize(nodeFs, p) }; // ---- git (R11) ---- -export function createGit(config) { - return function git({ repo, branch, dir, onLine }) { +const CLONE_MAX_MS = 5 * 60e3; +export function createGit(config, { spawnFn = spawn } = {}) { + return function git({ repo, branch, dir, onLine, timeoutMs = CLONE_MAX_MS }) { const b64 = Buffer.from(`x-access-token:${config.giteaToken}`).toString('base64'); const url = `${String(config.giteaUrl).replace(/\/+$/, '')}/${repo}.git`; - const args = ['-c', `http.extraHeader=Authorization: Basic ${b64}`, 'clone', '--depth', '1', '--branch', branch, '--', url, dir]; - const env = { PATH: process.env.PATH || '/usr/bin:/bin', GIT_TERMINAL_PROMPT: '0', GIT_CONFIG_NOSYSTEM: '1', HOME: tmpdir() }; + const args = ['-c', 'http.lowSpeedLimit=1000', '-c', 'http.lowSpeedTime=60', 'clone', '--depth', '1', '--branch', branch, '--', url, dir]; + const env = { PATH: process.env.PATH || '/usr/bin:/bin', GIT_TERMINAL_PROMPT: '0', GIT_CONFIG_NOSYSTEM: '1', HOME: tmpdir(), + // Token nur über die Umgebung des git-Prozesses, nie in argv/.git/config (R11) + GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'http.extraHeader', GIT_CONFIG_VALUE_0: `Authorization: Basic ${b64}` }; return new Promise((resolve, reject) => { - const child = spawn('git', args, { env, stdio: ['ignore', 'pipe', 'pipe'] }); + const child = spawnFn('git', args, { env, stdio: ['ignore', 'pipe', 'pipe'] }); + let timedOut = false; + const timer = setTimeout(() => { timedOut = true; try { child.kill('SIGKILL'); } catch { /* beendet */ } reject(new Error('git clone: Zeitlimit überschritten')); }, Math.max(1000, Math.min(timeoutMs, CLONE_MAX_MS))); const feed = (s) => { let buf = ''; s.setEncoding('utf8'); s.on('data', (d) => { const parts = (buf + d).split(/\r?\n|\r/); buf = parts.pop(); parts.forEach((l) => l && onLine?.(l)); }); s.on('end', () => buf && onLine?.(buf)); }; feed(child.stdout); feed(child.stderr); - child.on('error', reject); - child.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`git clone fehlgeschlagen (Exit-Code ${code})`)))); + child.on('error', (e) => { clearTimeout(timer); reject(e); }); + child.on('close', (code) => { clearTimeout(timer); if (!timedOut) { if (code === 0) resolve(); else reject(new Error(`git clone fehlgeschlagen (Exit-Code ${code})`)); } }); }); }; } @@ -78,7 +83,9 @@ export function createJobs({ config, docker, fs: fsOver = {}, git, mem = readMem // ---- Warteschlange mit genau einem aktiven Slot und Speicherwächter ---- function pump() { if (active || queue.length === 0) return; - if (mem() < config.minFreeBytes) { + let free = 0; + try { free = mem(); } catch { /* Fehler = zu wenig Speicher */ } + if (!(free >= config.minFreeBytes)) { if (!retry) { retry = true; setTimer(() => { retry = false; pump(); }, MEM_RETRY_MS); } return; } @@ -114,6 +121,8 @@ export function createJobs({ config, docker, fs: fsOver = {}, git, mem = readMem // ---- Container-Schritt ---- async function containerStep({ job, id, step, cmd, pm, workDir, cacheDir, env, onLine, deadline }) { await fs.mkdir(cacheDir, { recursive: true }); + const t = new Date(now()); + await fs.utimes(cacheDir, t, t).catch(() => {}); // Alter = letzte Nutzung await fs.chownR(workDir); await fs.chownR(cacheDir); // R7 const remaining = Math.max(1000, deadline - now()); @@ -125,42 +134,48 @@ export function createJobs({ config, docker, fs: fsOver = {}, git, mem = readMem // Klonen + Erkennung + Installation (R1); Aufrufer hält den Slot. async function prepare(job, { onLine, workDir, id = crypto.randomUUID(), deadline = now() + config.buildTimeoutMs }) { const cacheDir = join(config.cacheRoot, cacheName(job.repo)); - activeCaches.add(cacheName(job.repo)); - try { - await fs.rm(workDir, { recursive: true, force: true }); - await fs.mkdir(dirname(workDir), { recursive: true }); - await gitFn({ repo: job.repo, branch: job.branch, dir: workDir, onLine }); - const detected = detectFromDir(workDir, { mode: job.mode, index: job.index }); - if (detected.error || detected.kind === 'static' || !detected.install?.length) return { workDir, detected }; - const r = await containerStep({ job, id, step: 'install', cmd: detected.install, pm: detected.pm, workDir, cacheDir, onLine, deadline }); - if (r.code !== 0) return { workDir, detected, error: failText('Installation', r.code, config.buildTimeoutMs) }; - return { workDir, detected }; - } finally { - activeCaches.delete(cacheName(job.repo)); - } + await fs.rm(workDir, { recursive: true, force: true }); + await fs.mkdir(dirname(workDir), { recursive: true }); + await gitFn({ repo: job.repo, branch: job.branch, dir: workDir, onLine, timeoutMs: Math.max(1000, deadline - now()) }); + const detected = detectFromDir(workDir, { mode: job.mode, index: job.index }); + if (detected.error || detected.kind === 'static' || !detected.install?.length) return { workDir, detected }; + const r = await containerStep({ job, id, step: 'install', cmd: detected.install, pm: detected.pm, workDir, cacheDir, onLine, deadline }); + if (r.code !== 0) return { workDir, detected, error: failText('Installation', r.code, config.buildTimeoutMs) }; + return { workDir, detected }; + } + + // Cache für die gesamte Laufzeit vor pruneCache schützen. + async function holdCache(repo, fn) { + const n = cacheName(repo); + activeCaches.add(n); + try { return await fn(); } finally { activeCaches.delete(n); } } // ---- Atomare Veröffentlichung (R11) ---- - async function publish(job, srcDir, { skipGit }) { + async function publish(job, srcDir, { skipGit, onLine }) { const target = join(config.previewsRoot, job.sub); const fresh = `${target}.new`; const old = `${target}.old`; await fs.rm(fresh, { recursive: true, force: true }); + // Abgebrochener früherer Tausch: fehlt , ist .old die letzte gute Kopie – zurückholen. + const exists = (p) => fs.lstat(p).then(() => true, () => false); + if (!(await exists(target)) && (await exists(old))) await fs.rename(old, target); await fs.rm(old, { recursive: true, force: true }); + let hadOld = true; try { await fs.mkdir(config.previewsRoot, { recursive: true }); await fs.copyTree(fs, srcDir, fresh, { skipGit }); - let hadOld = true; try { await fs.rename(target, old); } catch (e) { if (e.code !== 'ENOENT') throw e; hadOld = false; } try { await fs.rename(fresh, target); } catch (e) { - if (hadOld) await fs.rename(old, target).catch(() => {}); + if (hadOld) await fs.rename(old, target).catch((e2) => onLine?.(`Rollback fehlgeschlagen: ${e2.message}`)); throw e; } - await fs.rm(old, { recursive: true, force: true }); } catch (e) { await fs.rm(fresh, { recursive: true, force: true }).catch(() => {}); throw e; } + // Ab hier ist der Preview live; Aufräumen darf den Lauf nicht scheitern lassen. + await fs.rm(old, { recursive: true, force: true }).catch((e) => onLine?.(`Aufräumen fehlgeschlagen: ${e.message}`)); } async function findOutput(workDir, dirs) { @@ -179,7 +194,8 @@ export function createJobs({ config, docker, fs: fsOver = {}, git, mem = readMem } // ---- Ablauf eines Build-Laufs ---- - async function execute(run, job) { + const execute = (run, job) => holdCache(job.repo, () => executeRun(run, job)); + async function executeRun(run, job) { const onLine = makeLog(run.log, masker(job)); const workDir = join(config.workRoot, 'build', run.runId); const deadline = now() + config.buildTimeoutMs; @@ -192,7 +208,7 @@ export function createJobs({ config, docker, fs: fsOver = {}, git, mem = readMem if (prep.error) throw new Error(prep.error); if (d.kind === 'static') { if (d.hint) onLine(d.hint); - await publish(job, workDir, { skipGit: true }); + await publish(job, workDir, { skipGit: true, onLine }); run.result = 'static'; } else if (d.kind === 'dev') { run.result = 'dev'; @@ -210,7 +226,7 @@ export function createJobs({ config, docker, fs: fsOver = {}, git, mem = readMem if (job.mode === 'auto' && d.devCmd) { onLine(`${problem} – Dev-Server wird verwendet`); run.result = 'dev'; } else throw new Error(problem); } else { - await publish(job, out, { skipGit: false }); + await publish(job, out, { skipGit: false, onLine }); run.result = 'build'; } } @@ -245,12 +261,13 @@ export function createJobs({ config, docker, fs: fsOver = {}, git, mem = readMem // Für Dev (Task 5): gleicher Slot, Klon + Installation in workDir. function prepareWorkspace(job, { onLine = () => {}, workDir } = {}) { - const log = (l) => onLine(masker(job)(l)); - return enqueue(async () => { + const mask = masker(job); + const log = (l) => onLine(mask(l)); + return enqueue(() => holdCache(job.repo, async () => { const r = await prepare(job, { onLine: log, workDir }); if (r.error) throw new Error(r.error); return { workDir: r.workDir, detected: r.detected }; - }); + })); } async function pruneCache() { diff --git a/runner/test/jobs.test.js b/runner/test/jobs.test.js index 20740c0..8aab786 100644 --- a/runner/test/jobs.test.js +++ b/runner/test/jobs.test.js @@ -3,7 +3,9 @@ import assert from 'node:assert/strict'; import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync, statSync, utimesSync, symlinkSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { createJobs } from '../src/jobs.js'; +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; +import { createJobs, createGit } from '../src/jobs.js'; const TOKEN = 'tok_SECRET_123456'; const B64 = Buffer.from(`x-access-token:${TOKEN}`).toString('base64'); @@ -161,3 +163,71 @@ test('pruneCache: > 14 Tage weg, dann älteste bis <= Limit', async () => { assert.deepEqual(readdirSync(s.config.cacheRoot).sort(), ['b__r', 'c__r']); rmSync(s.root, { recursive: true, force: true }); }); + +function fakeSpawn() { + const calls = []; + const fn = (cmd, args, opts) => { + const c = new EventEmitter(); c.stdout = new PassThrough(); c.stderr = new PassThrough(); c.killed = null; + c.kill = (sig) => { c.killed = sig; }; + calls.push({ cmd, args, opts, child: c }); + return c; + }; + return { fn, calls }; +} + +test('createGit: Token nicht in argv/URL, Umgebung minimal + GIT_CONFIG_*', async () => { + const sp = fakeSpawn(); + const git = createGit({ giteaUrl: 'https://git.x/', giteaToken: TOKEN }, { spawnFn: sp.fn }); + const p = git({ repo: 'own/rep', branch: 'main', dir: '/w/x' }); + const { args, opts, child } = sp.calls[0]; + assert.ok(!args.join(' ').includes(TOKEN) && !args.join(' ').includes(B64)); + assert.ok(args.includes('https://git.x/own/rep.git')); + assert.deepEqual(args.slice(args.indexOf('clone')), ['clone', '--depth', '1', '--branch', 'main', '--', 'https://git.x/own/rep.git', '/w/x']); + assert.ok(args.includes('http.lowSpeedLimit=1000') && args.includes('http.lowSpeedTime=60')); + assert.deepEqual(Object.keys(opts.env).sort(), ['GIT_CONFIG_COUNT', 'GIT_CONFIG_KEY_0', 'GIT_CONFIG_NOSYSTEM', 'GIT_CONFIG_VALUE_0', 'GIT_TERMINAL_PROMPT', 'HOME', 'PATH']); + assert.equal(opts.env.GIT_CONFIG_KEY_0, 'http.extraHeader'); + assert.equal(opts.env.GIT_CONFIG_VALUE_0, `Authorization: Basic ${B64}`); + child.emit('close', 0); + await p; +}); + +test('createGit: Zeitlimit beendet git mit SIGKILL und lehnt ab', async () => { + const sp = fakeSpawn(); + const git = createGit({ giteaUrl: 'https://git.x', giteaToken: TOKEN }, { spawnFn: sp.fn }); + await assert.rejects(git({ repo: 'own/rep', branch: 'main', dir: '/w/x', timeoutMs: 1000 }), /Zeitlimit/); + assert.equal(sp.calls[0].child.killed, 'SIGKILL'); +}); + +test('publish: fehlt und nur .old existiert, wird .old zurückgeholt (Build-Fehler danach)', async () => { + const s = setup({ files: { 'package.json': PKG() }, dockerImpl: (args) => ({ code: args.includes('build') ? 2 : 0 }) }); + // Zustand nach Absturz: nur demo.old; Erfolg-Pfad über static + mkdirSync(join(s.config.previewsRoot, 'demo.old')); writeFileSync(join(s.config.previewsRoot, 'demo.old', 'index.html'), 'GUT'); + const s2 = setup({ files: { 'index.html': 'NEU' } }); + mkdirSync(join(s2.config.previewsRoot, 'demo.old')); writeFileSync(join(s2.config.previewsRoot, 'demo.old', 'index.html'), 'GUT'); + const r = await wait(s2.jobs, s2.jobs.submit(job())); + assert.equal(r.state, 'done'); + assert.equal(readFileSync(join(s2.config.previewsRoot, 'demo', 'index.html'), 'utf8'), 'NEU'); +}); + +test('publish: Aufräumfehler nach dem Tausch lässt den Lauf nicht scheitern', async () => { + const root = mkdtempSync(join(tmpdir(), 'jobs-')); + const config = { giteaUrl: 'x', giteaToken: 't', previewsRoot: join(root, 'p'), workRoot: join(root, 'w'), cacheRoot: join(root, 'c'), minFreeBytes: 0, buildTimeoutMs: 1000 }; + mkdirSync(join(config.previewsRoot, 'demo'), { recursive: true }); + const jobs = createJobs({ config, docker: {}, git: async ({ dir }) => { mkdirSync(dir, { recursive: true }); writeFileSync(join(dir, 'index.html'), 'N'); }, mem: () => 1e12, + fs: { chownR: async () => {}, rm: async (p, o) => { if (p.endsWith('demo.old') && existsSync(p)) throw new Error('boom'); return (await import('node:fs/promises')).rm(p, o); } } }); + const r = await wait(jobs, jobs.submit(job())); + assert.equal(r.state, 'done'); + assert.equal(readFileSync(join(config.previewsRoot, 'demo', 'index.html'), 'utf8'), 'N'); + assert.ok(r.log.some((l) => /Aufräumen fehlgeschlagen/.test(l))); +}); + +test('mem() wirft ⇒ wie wenig Speicher; Cache während des Laufs vor pruneCache geschützt', async () => { + let boom = true; let s; let during; + s = setup({ files: { 'package.json': PKG() }, mem: () => { if (boom) throw new Error('x'); return 1e12; }, + dockerImpl: async () => { const d = join(s.config.cacheRoot, 'own__rep'); const t = (Date.now() - 40 * 864e5) / 1000; utimesSync(d, t, t); await s.jobs.pruneCache(); during = existsSync(d); return { code: 1 }; } }); + const id = s.jobs.submit(job({ mode: 'build' })); + assert.equal(s.jobs.get(id).state, 'queued'); + boom = false; s.timers[0].fn(); + await wait(s.jobs, id); + assert.equal(during, true); +});