From 96c26fb9b8532af9d372d1a6326afcba72f0389e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 21:39:49 +0000 Subject: [PATCH] =?UTF-8?q?fix(runner):=20Dev-Neustart=20bei=20ge=C3=A4nde?= =?UTF-8?q?rten=20Variablen,=20Liveness-Pr=C3=BCfung,=20npm-ci-Fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- runner/src/dev.js | 26 ++++++++++++++---- runner/src/docker.js | 8 +++++- runner/src/jobs.js | 7 ++++- runner/src/proxy.js | 3 ++- runner/test/dev.test.js | 55 +++++++++++++++++++++++++++++++++++++- runner/test/docker.test.js | 16 ++++++++++- runner/test/jobs.test.js | 23 ++++++++++++++++ runner/test/proxy.test.js | 11 ++++++++ 8 files changed, 139 insertions(+), 10 deletions(-) diff --git a/runner/src/dev.js b/runner/src/dev.js index 10ddc80..79e2fca 100644 --- a/runner/src/dev.js +++ b/runner/src/dev.js @@ -1,5 +1,6 @@ // Dev-Server bei Bedarf: Container mit `npm run dev` starten, bei Leerlauf stoppen, max. devMax gleichzeitig (LRU-Verdrängung). import net from 'node:net'; +import { createHash } from 'node:crypto'; import { join } from 'node:path'; import * as nodeFs from 'node:fs/promises'; import { devRunArgs } from './docker.js'; @@ -11,6 +12,9 @@ const POLL_MS = 1000; const RETRY_MS = 60e3; const UID = 1000; const PREFIX = 'preview-dev-'; +const LIVE_CHECK_MS = 10e3; +// I2: Stand der Startparameter (Variablen, Branch, Modus): Aenderung => Neustart +const cfgHash = (job) => createHash('sha256').update(JSON.stringify([job.branch, job.mode, Object.entries(job.env || {}).sort(([a], [b]) => (a < b ? -1 : 1))])).digest('hex'); const cacheName = (repo) => repo.replace('/', '__'); async function chownR(p) { @@ -126,7 +130,14 @@ export function createDev({ async function ensure(sub, job) { specs.set(sub, job); const cur = entries.get(sub); - if (cur && cur.sha === job.sha) { + // I7: ein abgestuerzter/beendeter Container darf nicht als online gelten + if (cur && cur.state === 'online' && docker.running && now() - (cur.checkedAt || 0) >= LIVE_CHECK_MS) { + let alive = true; + try { alive = await docker.running(PREFIX + sub); } catch { /* unklar: als lebendig behandeln */ } + cur.checkedAt = now(); + if (!alive && entries.get(sub) === cur && cur.state === 'online') { console.error('dev', sub, 'Container laeuft nicht mehr - Neustart'); await shutdown(cur, 'sleeping'); } + } + if (cur && cur.sha === job.sha && cur.cfg === cfgHash(job)) { if (cur.state === 'failed' && now() - cur.failedAt < RETRY_MS) return { state: 'failed' }; // nicht öfter als alle 60 s neu versuchen if (cur.state === 'online') { cur.lastUsed = now(); return { state: 'online', target: `http://${PREFIX}${sub}:${cur.port}` }; } if (cur.state === 'starting') { cur.lastUsed = now(); return { state: 'starting' }; } @@ -138,7 +149,7 @@ export function createDev({ if (cur) stops.push(shutdown(cur, 'sleeping'), cur.promise); // neuer sha oder failed/sleeping: frisch starten, erst nach dem alten Eintrag const others = [...entries.values()].filter((e) => e !== cur && isRunning(e)).sort((a, b) => a.lastUsed - b.lastUsed); while (others.length >= config.devMax) stops.push(shutdown(others.shift(), 'sleeping')); // LRU - const e = { sub, state: 'starting', sha: job.sha, lastUsed: now(), log: [], port: null, release: jobs.retainCache(job.repo), promise: null, cancelled: false, created: false, stopping: null, failedAt: 0 }; + const e = { sub, state: 'starting', sha: job.sha, cfg: cfgHash(job), checkedAt: now(), lastUsed: now(), log: [], port: null, release: jobs.retainCache(job.repo), promise: null, cancelled: false, created: false, stopping: null, failedAt: 0 }; entries.set(sub, e); e.promise = Promise.allSettled(stops).then(() => start(e, job)); return { state: 'starting' }; @@ -147,9 +158,9 @@ export function createDev({ // R12: Spezifikation merken, ohne zu starten. Läuft der sub mit anderem sha, wird nach den üblichen Regeln neu gestartet. async function register(sub, job) { const cur = entries.get(sub); - if (cur && isRunning(cur) && cur.sha !== job.sha) return ensure(sub, job); + if (cur && isRunning(cur) && (cur.sha !== job.sha || cur.cfg !== cfgHash(job))) return ensure(sub, job); specs.set(sub, job); - if (cur && cur.sha === job.sha && (cur.state === 'online' || cur.state === 'starting' || cur.state === 'failed')) return { state: cur.state }; + if (cur && cur.sha === job.sha && cur.cfg === cfgHash(job) && (cur.state === 'online' || cur.state === 'starting' || cur.state === 'failed')) return { state: cur.state }; return { state: 'sleeping' }; } // R12: Start mit der gemerkten Spezifikation (Proxy kennt nur die Subdomain). @@ -159,6 +170,11 @@ export function createDev({ return ensure(sub, job); } + // I7: Proxy meldet einen nicht erreichbaren Container: als schlafend markieren und entfernen (naechster Aufruf startet neu). + async function markDead(sub) { + const e = entries.get(sub); + if (e && e.state === 'online') await shutdown(e, 'sleeping'); + } const touch = (sub) => { const e = entries.get(sub); if (e) e.lastUsed = now(); }; async function stop(sub) { specs.delete(sub); // 'off': Spezifikation vergessen, bis das Portal neu registriert @@ -184,5 +200,5 @@ export function createDev({ } const idle = () => Promise.all([...entries.values()].map((e) => e.promise)).then(() => {}); - return { ensure, register, wake, touch, stop, list, status, sweep, cleanupOrphans, idle }; + return { ensure, register, wake, touch, markDead, stop, list, status, sweep, cleanupOrphans, idle }; } diff --git a/runner/src/docker.js b/runner/src/docker.js index 7591b5d..f7fc10f 100644 --- a/runner/src/docker.js +++ b/runner/src/docker.js @@ -177,10 +177,16 @@ export function createDocker({ exec = spawnDocker, baseEnv = process.env, envDir return { run, async ps(label) { - const { code, lines } = await capture(['ps', '--filter', `label=${label}`, '--format', '{{.Names}}']); + const { code, lines } = await capture(['ps', '-a', '--filter', `label=${label}`, '--format', '{{.Names}}']); if (code !== 0) throw new Error(`docker ps fehlgeschlagen (${code})`); return lines.map((l) => l.trim()).filter((l) => NAME_RE.test(l)); }, + // Laeuft der Container? (inspect; fehlt er oder ist beendet, dann false) + async running(name) { + checkName(name); + const { code, lines } = await capture(['inspect', '-f', '{{.State.Running}}', name]); + return code === 0 && lines.some((l) => l.trim() === 'true'); + }, async rm(name) { checkName(name); return capture(['rm', '-f', name]); }, async stop(name) { checkName(name); return capture(['rm', '-f', name]); }, }; diff --git a/runner/src/jobs.js b/runner/src/jobs.js index 588322f..cf88bde 100644 --- a/runner/src/jobs.js +++ b/runner/src/jobs.js @@ -139,7 +139,12 @@ export function createJobs({ config, docker, fs: fsOver = {}, git, mem = readMem await gitFn({ repo: job.repo, branch: job.branch, dir: workDir, onLine, timeoutMs: Math.max(1000, deadline - now()) }); const detected = detectFromDir(workDir, { mode: job.mode, index: job.index }); if (detected.error || detected.kind === 'static' || !detected.install?.length) return { workDir, detected }; - const r = await containerStep({ job, id, step: 'install', cmd: detected.install, pm: detected.pm, workDir, cacheDir, onLine, deadline }); + let r = await containerStep({ job, id, step: 'install', cmd: detected.install, pm: detected.pm, workDir, cacheDir, onLine, deadline }); + // I8: `npm ci` scheitert bei nicht passender Lock-Datei; wie der bisherige Build-Weg auf `npm install` ausweichen. + if (r.code !== 0 && detected.pm === 'npm' && detected.install[1] === 'ci') { + onLine('npm ci fehlgeschlagen – Wiederholung mit npm install'); + r = await containerStep({ job, id, step: 'install2', cmd: ['npm', 'install', '--include=dev', '--no-audit', '--no-fund'], pm: detected.pm, workDir, cacheDir, onLine, deadline }); + } if (r.code !== 0) return { workDir, detected, error: failText('Installation', r.code, config.buildTimeoutMs) }; return { workDir, detected }; } diff --git a/runner/src/proxy.js b/runner/src/proxy.js index cd7588d..7cc79c2 100644 --- a/runner/src/proxy.js +++ b/runner/src/proxy.js @@ -83,7 +83,8 @@ export function createProxy({ config, dev, request = http.request, upstreamTimeo }); let timedOut = false; up.setTimeout(upstreamTimeoutMs, () => { timedOut = true; up.destroy(); }); - up.on('error', () => { + up.on('error', (err) => { + if (['ECONNREFUSED', 'ENOTFOUND', 'EHOSTUNREACH', 'EAI_AGAIN'].includes(err?.code)) dev.markDead?.(sub)?.catch?.(() => {}); if (!res.headersSent) { const body = timedOut ? 'Die Preview antwortet nicht rechtzeitig (504).' : 'Die Preview ist gerade nicht erreichbar (502).'; res.writeHead(timedOut ? 504 : 502, { 'Content-Type': 'text/plain; charset=utf-8', 'Content-Length': Buffer.byteLength(body) }); diff --git a/runner/test/dev.test.js b/runner/test/dev.test.js index 8dfa31d..e85e547 100644 --- a/runner/test/dev.test.js +++ b/runner/test/dev.test.js @@ -5,7 +5,7 @@ import { createDev } from '../src/dev.js'; const MIN = 60e3; const GB = 1024 ** 3; -function setup({ mem = 10 * GB, connect, detected, prep, rmImpl } = {}) { +function setup({ mem = 10 * GB, connect, detected, prep, rmImpl, alive } = {}) { const config = { workRoot: '/w/work', cacheRoot: '/w/cache', baseHost: 'project.webklar.com', minFreeBytes: 1.5 * GB, devIdleMs: 20 * MIN, devMax: 3, devReadyMs: 180e3, giteaToken: 'tok_GEHEIM' }; const clock = { t: 1_000_000 }; const rmDirs = []; const runs = []; const removed = []; const held = new Map(); const prepared = []; @@ -14,6 +14,7 @@ function setup({ mem = 10 * GB, connect, detected, prep, rmImpl } = {}) { run: async (args, opts) => { if (args[0] === 'run') runs.push({ args, opts }); else opts?.onLine?.('container-log-zeile', 'out'); return { code: 0 }; }, rm: async (n) => { removed.push(n); if (rmImpl) await rmImpl(n); return { code: 0, lines: [] }; }, ps: async () => psResult, + ...(alive ? { running: async (n) => alive(n) } : {}), }; const jobs = { prepareWorkspace: async (job, o) => { prepared.push({ job, o }); if (prep) return prep(job, o); o.onLine?.('npm install ok'); return { workDir: o.workDir, detected: detected ?? { pm: 'npm', devCmd: ['npm', 'run', 'dev', '--', '--host', '0.0.0.0', '--port', '5173'], devPort: 5173 } }; }, @@ -291,3 +292,55 @@ test('register: schlafender sub mit neuem sha startet nicht, wake nutzt das neue assert.equal(s.runs.length, 2); assert.equal(s.prepared[1].job.sha, '7654321'); }); + +test('I2: gleicher sha, geänderte Variablen ⇒ register/ensure starten neu', async () => { + const s = setup(); + await s.dev.ensure('a', job('a')); await s.dev.idle(); + assert.equal(s.runs.length, 1); + // gleiche Variablen (auch andere Reihenfolge): kein Neustart + assert.equal((await s.dev.register('a', job('a', { env: { API_KEY: 'geheim-wert' } }))).state, 'online'); + assert.equal(s.runs.length, 1); + assert.equal((await s.dev.register('a', job('a', { env: { API_KEY: 'neuer-wert' } }))).state, 'starting'); + await s.dev.idle(); + assert.equal(s.runs.length, 2); + assert.ok(s.removed.includes('preview-dev-a')); + assert.equal(s.runs[1].opts.env.API_KEY, 'neuer-wert'); + // Branch-Wechsel ebenfalls + await s.dev.ensure('a', job('a', { env: { API_KEY: 'neuer-wert' }, branch: 'dev' })); await s.dev.idle(); + assert.equal(s.runs.length, 3); +}); + +test('I7: beendeter Container ⇒ sleeping, entfernt und per wake neu gestartet', async () => { + let up = true; + const s = setup({ alive: () => up }); + await s.dev.wake('a').catch(() => {}); // unbekannt + await s.dev.register('a', job('a')); + await s.dev.wake('a'); await s.dev.idle(); + assert.equal(s.runs.length, 1); + up = false; + s.clock.t += 11e3; + assert.deepEqual(await s.dev.wake('a'), { state: 'starting' }); + await s.dev.idle(); + assert.ok(s.removed.includes('preview-dev-a')); + assert.equal(s.runs.length, 2); +}); + +test('I7: Lebendigkeitsprüfung höchstens alle 10 s', async () => { + let checks = 0; + const s = setup({ alive: () => { checks++; return true; } }); + await s.dev.ensure('a', job('a')); await s.dev.idle(); + for (let i = 0; i < 5; i++) await s.dev.ensure('a', job('a')); + assert.equal(checks, 0); + s.clock.t += 11e3; + await s.dev.ensure('a', job('a')); await s.dev.ensure('a', job('a')); + assert.equal(checks, 1); +}); + +test('I7: markDead entfernt den Container und setzt sleeping', async () => { + const s = setup(); + await s.dev.ensure('a', job('a')); await s.dev.idle(); + await s.dev.markDead('a'); + assert.ok(s.removed.includes('preview-dev-a')); + assert.equal(s.dev.list().find((e) => e.sub === 'a').state, 'sleeping'); + assert.equal((await s.dev.wake('a')).state, 'starting'); +}); diff --git a/runner/test/docker.test.js b/runner/test/docker.test.js index bcd0522..728a234 100644 --- a/runner/test/docker.test.js +++ b/runner/test/docker.test.js @@ -156,7 +156,7 @@ test('ps/stop/rm', async () => { }; const docker = createDocker({ exec }); assert.deepEqual(await docker.ps('webklar.preview'), ['preview-dev-a', 'preview-dev-b']); - assert.deepEqual(calls[0], ['ps', '--filter', 'label=webklar.preview', '--format', '{{.Names}}']); + assert.deepEqual(calls[0], ['ps', '-a', '--filter', 'label=webklar.preview', '--format', '{{.Names}}']); await docker.stop('preview-dev-a'); await docker.rm('preview-dev-a'); assert.deepEqual(calls[1], ['rm', '-f', 'preview-dev-a']); @@ -207,3 +207,17 @@ test('Zeitlimit mit env-file: Datei wird entfernt', async () => { assert.deepEqual(await docker.run(['run', '--name', 'j', 'node:20'], { env: { K: 'v' }, timeoutMs: 10 }), { code: 124 }); assert.equal(existsSync(filePath), false); }); + +test('ps: mit -a, damit beendete Container aufgeräumt werden', async () => { + const calls = []; + const exec = (args) => { calls.push(args); const c = fakeChild(); setImmediate(() => { c.stdout.end(); c.stderr.end(); c.emit('close', 0); }); return c; }; + await createDocker({ exec }).ps('webklar.preview'); + assert.deepEqual(calls[0].slice(0, 2), ['ps', '-a']); +}); + +test('running: inspect true/false/fehlend', async () => { + const mk = (out, code) => () => { const c = fakeChild(); setImmediate(() => { if (out) c.stdout.write(out); c.stdout.end(); c.stderr.end(); c.emit('close', code); }); return c; }; + assert.equal(await createDocker({ exec: mk('true\n', 0) }).running('preview-dev-a'), true); + assert.equal(await createDocker({ exec: mk('false\n', 0) }).running('preview-dev-a'), false); + assert.equal(await createDocker({ exec: mk('', 1) }).running('preview-dev-a'), false); +}); diff --git a/runner/test/jobs.test.js b/runner/test/jobs.test.js index 8aab786..a8f56ed 100644 --- a/runner/test/jobs.test.js +++ b/runner/test/jobs.test.js @@ -231,3 +231,26 @@ test('mem() wirft ⇒ wie wenig Speicher; Cache während des Laufs vor pruneCach await wait(s.jobs, id); assert.equal(during, true); }); + +test('I8: npm ci scheitert ⇒ Wiederholung mit npm install (Protokoll, Build läuft weiter)', async () => { + const s = setup({ + files: { 'package.json': PKG(), 'package-lock.json': '{}' }, + dockerImpl: (args) => { + if (args.includes('ci')) return { code: 1 }; + if (args.includes('build')) { mkdirSync(join(workOf(args), 'dist'), { recursive: true }); writeFileSync(join(workOf(args), 'dist', 'index.html'), 'N'); } + return { code: 0 }; + }, + }); + const r = await wait(s.jobs, s.jobs.submit(job())); + assert.equal(r.state, 'done'); assert.equal(r.result, 'build'); + assert.equal(s.calls.length, 3); + assert.deepEqual(s.calls[1].args.slice(-5), ['npm', 'install', '--include=dev', '--no-audit', '--no-fund']); + assert.ok(r.log.some((l) => l.includes('npm install'))); +}); + +test('I8: auch npm install scheitert ⇒ Installationsfehler', async () => { + const s = setup({ files: { 'package.json': PKG(), 'package-lock.json': '{}' }, dockerImpl: () => ({ code: 1 }) }); + const r = await wait(s.jobs, s.jobs.submit(job())); + assert.equal(r.state, 'failed'); assert.match(r.error, /Installation fehlgeschlagen/); + assert.equal(s.calls.length, 2); +}); diff --git a/runner/test/proxy.test.js b/runner/test/proxy.test.js index 292f2c5..1ceff81 100644 --- a/runner/test/proxy.test.js +++ b/runner/test/proxy.test.js @@ -221,3 +221,14 @@ test('WebSocket-Daten touchen höchstens einmal pro Minute', async () => { }, { now: () => clock.t }); } finally { t.srv.closeAllConnections?.(); await new Promise((r) => t.srv.close(r)); } }); + +test('I7: Upstream ECONNREFUSED ⇒ dev.markDead + 502', async () => { + const dead = []; + const srv = http.createServer(); await new Promise((r) => srv.listen(0, '127.0.0.1', r)); + const port = srv.address().port; await new Promise((r) => srv.close(r)); // Port ist jetzt zu + await withProxy({ wake: async () => ({ state: 'online', target: `http://127.0.0.1:${port}` }), markDead: async (s) => { dead.push(s); } }, async ({ port: p }) => { + const { res } = await get(p, 'demo.project.webklar.com'); + assert.equal(res.statusCode, 502); + assert.deepEqual(dead, ['demo']); + }); +});