diff --git a/runner/src/dev.js b/runner/src/dev.js index 44062b1..70dd6ed 100644 --- a/runner/src/dev.js +++ b/runner/src/dev.js @@ -8,6 +8,7 @@ import { readMemAvailable } from './server.js'; const LOG_LINES = 300; const MAX_LINE = 2000; const POLL_MS = 1000; +const RETRY_MS = 60e3; const UID = 1000; const PREFIX = 'preview-dev-'; const cacheName = (repo) => repo.replace('/', '__'); @@ -39,6 +40,7 @@ export function tcpConnect(host, port, timeoutMs = 2000) { export function createDev({ config, docker, jobs, mem = readMemAvailable, now = Date.now, connect = tcpConnect, setTimer = (fn, ms) => setTimeout(fn, ms).unref?.() ?? 0, prepareDirs = defaultPrepareDirs, + rmDir = (p) => nodeFs.rm(p, { recursive: true, force: true }), }) { const entries = new Map(); // sub ⇒ { sub, state, sha, lastUsed, log, port, job, release, promise, cancelled } const sleep = (ms) => new Promise((r) => setTimer(r, ms)); @@ -56,8 +58,16 @@ export function createDev({ async function shutdown(e, finalState) { e.cancelled = true; e.release?.(); e.release = null; - if (finalState === 'off') entries.delete(e.sub); else { e.state = finalState; e.port = null; } - try { await docker.rm(PREFIX + e.sub); } catch (err) { console.error('dev rm', e.sub, err); } + if (finalState === 'off') { if (entries.get(e.sub) === e) entries.delete(e.sub); } else { e.state = finalState; e.port = null; } + // Nur der Eintrag, der den Container angelegt hat, entfernt ihn (Name ist je sub geteilt). Läuft `docker run` noch, + // räumt start() nach der Rückkehr selbst auf. + if (!e.stopping) { + e.stopping = (async () => { + if (!e.created) return; + try { const r = await docker.rm(PREFIX + e.sub); if (r && r.code) console.error('dev rm', e.sub, 'Exit-Code', r.code); } catch (err) { console.error('dev rm', e.sub, err); } + })(); + } + return e.stopping; } async function start(e, job) { @@ -69,12 +79,14 @@ export function createDev({ log(msg); e.release?.(); e.release = null; e.state = 'failed'; e.port = null; - try { await docker.rm(name); } catch { /* egal */ } + e.failedAt = now(); + if (e.created) { try { await docker.rm(name); } catch { /* egal */ } } }; try { - e.release = jobs.retainCache(job.repo); + if (e.cancelled) return; // vor dem Start abgebrochen: nichts einreihen (ein laufendes prepareWorkspace ließe sich nicht abbrechen) const workDir = join(config.workRoot, 'dev', e.sub); const cacheDir = join(config.cacheRoot, cacheName(job.repo)); + // Bleibt ein Rest: ist prepareWorkspace schon gestartet, belegt es den Build-Slot bis zum Ende, auch nach Abbruch. const prep = await jobs.prepareWorkspace(job, { onLine: log, workDir }); if (e.cancelled) return; const d = prep.detected; @@ -88,6 +100,7 @@ export function createDev({ }); // Feste Variable zuletzt, damit Projektvariablen sie nicht überschreiben; Werte laufen per --env-file (R10). const env = { ...job.env, __VITE_ADDITIONAL_SERVER_ALLOWED_HOSTS: `.${config.baseHost}` }; + e.created = true; const r = await docker.run(args, { env, onLine: (l) => log(l) }); if (e.cancelled) { await docker.rm(name).catch(() => {}); return; } if (r.code !== 0) return await fail(`Container konnte nicht gestartet werden (Exit-Code ${r.code})`); @@ -112,6 +125,7 @@ export function createDev({ async function ensure(sub, job) { const cur = entries.get(sub); if (cur && cur.sha === job.sha) { + if (cur.state === 'failed' && now() - cur.failedAt < RETRY_MS) return { state: 'failed' }; // nicht öfter als alle 60 s neu versuchen if (cur.state === 'online') { cur.lastUsed = now(); return { state: 'online', target: `http://${PREFIX}${sub}:${cur.port}` }; } if (cur.state === 'starting') { cur.lastUsed = now(); return { state: 'starting' }; } } @@ -119,17 +133,23 @@ export function createDev({ try { free = mem(); } catch { /* Fehler = zu wenig Speicher */ } if (!(free >= config.minFreeBytes)) throw Object.assign(new Error('Server ausgelastet'), { status: 503 }); const stops = []; - if (cur) stops.push(shutdown(cur, 'sleeping')); // neuer sha oder failed/sleeping: frisch starten + if (cur) stops.push(shutdown(cur, 'sleeping'), cur.promise); // neuer sha oder failed/sleeping: frisch starten, erst nach dem alten Eintrag const others = [...entries.values()].filter((e) => e !== cur && isRunning(e)).sort((a, b) => a.lastUsed - b.lastUsed); while (others.length >= config.devMax) stops.push(shutdown(others.shift(), 'sleeping')); // LRU - const e = { sub, state: 'starting', sha: job.sha, lastUsed: now(), log: [], port: null, release: null, promise: null, cancelled: false }; + const e = { sub, state: 'starting', sha: job.sha, lastUsed: now(), log: [], port: null, release: jobs.retainCache(job.repo), promise: null, cancelled: false, created: false, stopping: null, failedAt: 0 }; entries.set(sub, e); - e.promise = Promise.all(stops).then(() => start(e, job)); + e.promise = Promise.allSettled(stops).then(() => start(e, job)); return { state: 'starting' }; } const touch = (sub) => { const e = entries.get(sub); if (e) e.lastUsed = now(); }; - async function stop(sub) { const e = entries.get(sub); if (e) await shutdown(e, 'off'); } + async function stop(sub) { + const e = entries.get(sub); + if (!e) return; + await shutdown(e, 'off'); + await e.promise.catch(() => {}); + if (!entries.has(sub)) await rmDir(join(config.workRoot, 'dev', sub)).catch(() => {}); // Arbeitsordner aufräumen + } const list = () => [...entries.values()].map((e) => ({ sub: e.sub, state: e.state, lastUsed: e.lastUsed, sha: e.sha })); function status(sub) { const e = entries.get(sub); @@ -141,7 +161,8 @@ export function createDev({ } // Zustand liegt nur im Speicher: nach Runner-Neustart übrig gebliebene Dev-Container entfernen. async function cleanupOrphans() { - for (const n of await docker.ps('webklar.preview')) if (n.startsWith(PREFIX)) await docker.rm(n); + const names = (await docker.ps('webklar.preview')).filter((n) => n.startsWith(PREFIX)); + for (const r of await Promise.allSettled(names.map((n) => docker.rm(n)))) if (r.status === 'rejected') console.error('orphan rm', r.reason); } const idle = () => Promise.all([...entries.values()].map((e) => e.promise)).then(() => {}); diff --git a/runner/test/dev.test.js b/runner/test/dev.test.js index 5e2d0d0..59c3b7d 100644 --- a/runner/test/dev.test.js +++ b/runner/test/dev.test.js @@ -5,14 +5,14 @@ import { createDev } from '../src/dev.js'; const MIN = 60e3; const GB = 1024 ** 3; -function setup({ mem = 10 * GB, connect, detected, prep } = {}) { +function setup({ mem = 10 * GB, connect, detected, prep, rmImpl } = {}) { const config = { workRoot: '/w/work', cacheRoot: '/w/cache', baseHost: 'project.webklar.com', minFreeBytes: 1.5 * GB, devIdleMs: 20 * MIN, devMax: 3, devReadyMs: 180e3, giteaToken: 'tok_GEHEIM' }; const clock = { t: 1_000_000 }; - const runs = []; const removed = []; const held = new Map(); const prepared = []; + const rmDirs = []; const runs = []; const removed = []; const held = new Map(); const prepared = []; let psResult = []; const docker = { run: async (args, opts) => { if (args[0] === 'run') runs.push({ args, opts }); else opts?.onLine?.('container-log-zeile', 'out'); return { code: 0 }; }, - rm: async (n) => { removed.push(n); return { code: 0, lines: [] }; }, + rm: async (n) => { removed.push(n); if (rmImpl) await rmImpl(n); return { code: 0, lines: [] }; }, ps: async () => psResult, }; const jobs = { @@ -25,8 +25,9 @@ function setup({ mem = 10 * GB, connect, detected, prep } = {}) { connect: connect ?? (async (h, p) => { connects.push([h, p]); return true; }), setTimer: (fn, ms) => { clock.t += ms; queueMicrotask(fn); return 0; }, prepareDirs: async () => {}, + rmDir: async (p) => { rmDirs.push(p); }, }); - return { dev, clock, runs, removed, held, prepared, connects, config, setPs: (v) => { psResult = v; } }; + return { rmDirs, dev, clock, runs, removed, held, prepared, connects, config, setPs: (v) => { psResult = v; } }; } const job = (sub, o = {}) => ({ sub, repo: `own/${sub}`, sha: 'abcdef1', branch: 'main', mode: 'dev', env: { API_KEY: 'geheim-wert' }, ...o }); @@ -158,13 +159,13 @@ test('Cache wird gehalten solange der Container läuft, stop gibt frei', async ( assert.deepEqual(s.dev.list(), []); }); -test('stop während des Starts: Container entfernt, nie online', async () => { +test('stop während des Starts: nie online, kein Container', async () => { const s = setup(); await s.dev.ensure('a', job('a')); await s.dev.stop('a'); await s.dev.idle(); assert.equal(s.dev.status('a').state, 'off'); assert.equal(s.held.get('own/a'), 0); - assert.ok(s.removed.includes('preview-dev-a')); + assert.equal(s.runs.length, 0); // prepare läuft hier evtl. schon (nicht abbrechbar), der Container nie }); test('Protokoll maskiert Variablenwerte und ist auf 300 Zeilen begrenzt', async () => { @@ -182,3 +183,69 @@ test('Start: verwaiste preview-dev-*-Container werden entfernt', async () => { await s.dev.cleanupOrphans(); assert.deepEqual(s.removed, ['preview-dev-x', 'preview-dev-y']); }); + +const tick = () => new Promise((r) => setImmediate(r)); + +test('stop während Verdrängungs-rm: kein Cache-Leck, kein prepare, Arbeitsordner entfernt', async () => { + let gate; const opened = new Promise((r) => { gate = r; }); + let slow = false; + const s = setup({ rmImpl: async () => { if (slow) await opened; } }); + for (const n of ['a', 'b', 'c']) { await s.dev.ensure(n, job(n)); await s.dev.idle(); s.clock.t += MIN; } + slow = true; + await s.dev.ensure('d', job('d')); // verdrängt a, wartet auf dessen rm + await tick(); + const stopping = s.dev.stop('d'); + gate(); await stopping; await s.dev.idle(); + assert.equal(s.held.get('own/d'), 0); + assert.ok(!s.prepared.some((p) => p.job.sub === 'd')); + assert.equal(s.dev.status('d').state, 'off'); + assert.ok(s.rmDirs.includes('/w/work/dev/d')); +}); + +test('sha-Wechsel während des Starts: genau ein Container des neuen sha, alter entfernt neuen nie', async () => { + let release; const gate = new Promise((r) => { release = r; }); + let first = true; + const s = setup({ prep: async (j, o) => { if (first) { first = false; await gate; } return { workDir: o.workDir, detected: { pm: 'npm', devCmd: ['npm', 'run', 'dev'], devPort: 5173 } }; } }); + await s.dev.ensure('a', job('a')); + await s.dev.ensure('a', job('a', { sha: '1234567' })); + release(); await s.dev.idle(); + assert.equal(s.runs.length, 1); + assert.equal(s.removed.length, 0); // alter Eintrag hat nie einen Container angelegt + assert.equal(s.dev.status('a').sha, '1234567'); assert.equal(s.dev.status('a').state, 'online'); + assert.equal(s.held.get('own/a'), 1); +}); + +test('gleichzeitig sweep + ensure(neuer sha): ein Container, neuer bleibt online', async () => { + const s = setup(); + await s.dev.ensure('a', job('a')); await s.dev.idle(); + s.clock.t += 21 * MIN; + const p = Promise.all([s.dev.sweep(), s.dev.ensure('a', job('a', { sha: '1234567' }))]); + await p; await s.dev.idle(); + assert.equal(s.runs.length, 2); + assert.equal(s.dev.status('a').state, 'online'); assert.equal(s.dev.status('a').sha, '1234567'); + assert.equal(s.held.get('own/a'), 1); + // das letzte rm gehört dem alten Container und liegt vor dem zweiten run + assert.ok(s.removed.every((n) => n === 'preview-dev-a')); +}); + +test('failed-Eintrag mit gleichem sha: Neuversuch frühestens nach 60 s', async () => { + let ok = false; + const s = setup({ connect: async () => { if (!ok) throw new Error('nein'); return true; } }); + await s.dev.ensure('a', job('a')); await s.dev.idle(); + assert.equal(s.dev.status('a').state, 'failed'); + const prepared = s.prepared.length; + assert.deepEqual(await s.dev.ensure('a', job('a')), { state: 'failed' }); + assert.equal(s.prepared.length, prepared); + s.clock.t += 61e3; ok = true; + assert.deepEqual(await s.dev.ensure('a', job('a')), { state: 'starting' }); + await s.dev.idle(); + assert.equal(s.dev.status('a').state, 'online'); +}); + +test('cleanupOrphans: ein fehlschlagendes rm stoppt die übrigen nicht', async () => { + const s = setup({ rmImpl: async (n) => { if (n === 'preview-dev-x') throw new Error('kaputt'); } }); + s.setPs(['preview-dev-x', 'preview-dev-y']); + const orig = console.error; console.error = () => {}; + try { await s.dev.cleanupOrphans(); } finally { console.error = orig; } + assert.deepEqual(s.removed, ['preview-dev-x', 'preview-dev-y']); +});