From 5c39640224858833671764cecdc840cadc1e7882 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 10:29:31 +0000 Subject: [PATCH] =?UTF-8?q?Neue=20Repos=20importieren,=20Abgleich=20ohne?= =?UTF-8?q?=20Subdomains,=20Selbstheilung=20nur=20f=C3=BCr=20gehostete=20P?= =?UTF-8?q?rojekte?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- server/services/giteaAdmin.js | 105 ++++++++++++++++++++++---------- test/giteaAdmin.test.js | 110 ++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+), 32 deletions(-) create mode 100644 test/giteaAdmin.test.js diff --git a/server/services/giteaAdmin.js b/server/services/giteaAdmin.js index 8e1d9e7..157b5a6 100644 --- a/server/services/giteaAdmin.js +++ b/server/services/giteaAdmin.js @@ -15,6 +15,13 @@ import { } from './previewDeploy.js' import { giteaFetch } from './giteaClient.js' import { slugify } from './slug.js' +import { withLocks } from './keyedLock.js' +import { repoLockKey } from './repoMove.js' + +// Sicherheitsnetz wie bei listRecentPushes (siehe pushHistory.js, Controller-Ruling R8): sollte +// Gitea "page" ignorieren und immer dieselbe volle Seite liefern, blaettert die Schleife nicht +// endlos weiter (40 x 50 = 2000 Repos). +const MAX_REPO_PAGES = 40 export async function listAllAccessibleRepos() { const repos = [] @@ -27,40 +34,67 @@ export async function listAllAccessibleRepos() { if (data.length === 0) break repos.push(...data) if (data.length < limit) break + if (page >= MAX_REPO_PAGES) { + console.warn( + `[giteaAdmin] mehr als ${MAX_REPO_PAGES} Seiten Repos – weitere Repos werden nicht gelistet` + ) + break + } page += 1 } return repos } -async function resolveSubdomainForSync(repo, existingDoc) { - if (existingDoc?.subdomain) return existingDoc.subdomain +/** + * Findet Repos, die in Gitea liegen, aber noch kein Appwrite-Projektdokument haben, und legt fuer + * jedes nicht archivierte ein minimales Dokument an (status "synced", ohne Hosting/Subdomain) - + * ohne dass dafuer manuell "Abgleich" gedrueckt werden muesste. Wird vom Projekt-Dialog bei jedem + * Oeffnen aufgerufen, laeuft also oft: die aeussere `withLocks(['import'])`-Sperre serialisiert + * parallele Laeufe komplett (ein zweiter Aufruf sieht danach garantiert den Stand des ersten), die + * innere `withLocks([repoLockKey(...)])` je neuem Repo schuetzt zusaetzlich gegen andere + * gleichzeitige Schreiber desselben Projektdokuments (z. B. Umzug/Zuweisung). + */ +export async function importNewRepos() { + return withLocks(['import'], async () => { + const [repos, docs] = await Promise.all([ + listAllAccessibleRepos(), + listDocuments(config.collections.websiteProjects, [Query.limit(5000)]), + ]) - const candidate = slugify(repo.name) || slugify(repo.full_name?.split('/').pop()) - if (!candidate) return '' + // Ohne Ruecksicht auf Gross-/Kleinschreibung vergleichen (Controller-Ruling). + const existingNames = new Set( + docs.map((d) => String(d.repoFullName || '').toLowerCase()).filter(Boolean) + ) - const conflict = await listDocuments(config.collections.websiteProjects, [ - Query.equal('subdomain', candidate), - Query.limit(1), - ]) + let created = 0 + for (const repo of repos) { + const fullName = repo.full_name + if (!fullName || repo.archived) continue + if (existingNames.has(fullName.toLowerCase())) continue - if (!conflict[0] || conflict[0].repoFullName === repo.full_name) { - return candidate - } + await withLocks([repoLockKey(fullName)], async () => { + const recheck = await listDocuments(config.collections.websiteProjects, [ + Query.equal('repoFullName', fullName), + Query.limit(1), + ]) + if (recheck[0]) return - const ownerSlug = slugify(repo.full_name?.replace('/', '-')) - if (!ownerSlug) return '' + await upsertWebsiteProjectByRepo(fullName, { + projectName: repo.description || repo.name || fullName.split('/').pop(), + giteaRepoUrl: repo.html_url, + giteaRepoName: repo.name, + repoFullName: fullName, + archived: false, + status: 'synced', + giteaSyncedAt: new Date().toISOString(), + }) + created += 1 + }) + } - const ownerConflict = await listDocuments(config.collections.websiteProjects, [ - Query.equal('subdomain', ownerSlug), - Query.limit(1), - ]) - - if (!ownerConflict[0] || ownerConflict[0].repoFullName === repo.full_name) { - return ownerSlug - } - - return '' + return { created } + }) } export async function syncReposToAppwrite() { @@ -77,10 +111,6 @@ export async function syncReposToAppwrite() { Query.limit(1), ]) - const branch = repo.default_branch || 'main' - const subdomain = await resolveSubdomainForSync(repo, existing[0]) - const previewUrl = subdomain ? buildPreviewUrl(subdomain) : '' - const payload = { projectName: repo.description || repo.name || repoFullName.split('/').pop(), giteaRepoUrl: repo.html_url || `${config.gitea.baseUrl}/${repoFullName}`, @@ -90,9 +120,6 @@ export async function syncReposToAppwrite() { archived: Boolean(repo.archived), } - if (subdomain) payload.subdomain = subdomain - if (previewUrl) payload.previewUrl = previewUrl - if (!existing[0]) { payload.status = 'synced' created += 1 @@ -186,6 +213,20 @@ async function updatePreviewConfigFile(repoFullName, previewConfig, branch = 'ma }) } +/** + * true, wenn zu diesem Projekt wirklich ein Hosting gehoert (Preview/Website/Vorlage, oder ein + * bereits deployter bzw. anstehender Stand) - im Unterschied zu einem nur importierten/ + * abgeglichenen Repo ohne eigenes Hosting (z. B. status "synced" ohne Website). Bestimmt u. a., + * ob die Selbstheilung (ensureRepoPreviewConfig) fuer dieses Projekt ueberhaupt aktiv werden darf - + * sonst wuerde mit dem organisationsweiten Webhook jeder Push in ein bloss abgeglichenes Repo eine + * Preview anlegen. + */ +export function isHostedProject(doc) { + if (['preview', 'website', 'template'].includes(doc?.projectType)) return true + if (['deployed', 'pending'].includes(doc?.status)) return true + return false +} + /** * Selbstheilung fuer eine geloeschte/fehlende .webklar-preview.json: * Rekonstruiert die Hosting-Config aus dem in Appwrite gespeicherten @@ -201,11 +242,11 @@ export async function ensureRepoPreviewConfig(repoFullName, branch = 'main') { ]) const doc = docs[0] if (!doc || doc.archived) return null + if (!isHostedProject(doc)) return null const type = deriveProjectType(doc) - const hasWebsite = type === 'preview' || type === 'website' || type === 'template' const subdomain = doc.subdomain || '' - if (!hasWebsite || !subdomain) return null + if (!subdomain) return null const previewConfig = { enabled: true, diff --git a/test/giteaAdmin.test.js b/test/giteaAdmin.test.js new file mode 100644 index 0000000..8b2c6e3 --- /dev/null +++ b/test/giteaAdmin.test.js @@ -0,0 +1,110 @@ +import { test, before, after } from 'node:test' +import assert from 'node:assert/strict' +import { startFakeGitea } from './helpers/fakeGitea.js' +import { startFakeAppwrite } from './helpers/fakeAppwrite.js' + +let gitea +let aw +let config +let listAllAccessibleRepos +let importNewRepos +let syncReposToAppwrite +let isHostedProject +let ensureRepoPreviewConfig + +before(async () => { + gitea = await startFakeGitea({ + repos: [ + { full_name: 'knso/alt' }, + { full_name: 'JUSN/neu', description: 'Döner Nation' }, + { full_name: 'JUSN/archiv', archived: true }, + { full_name: 'knso/privat', private: true }, + { full_name: 'knso/synced-sub' }, + { full_name: 'knso/lead' }, + ], + }) + + aw = await startFakeAppwrite({ + seed: { + websiteProjects: [ + { repoFullName: 'knso/alt', projectName: 'alt', subdomain: 'alt' }, + { repoFullName: 'knso/synced-sub', projectName: 'synced-sub', status: 'synced', subdomain: 'synced-sub' }, + { repoFullName: 'knso/lead', projectName: 'lead', status: 'deployed', subdomain: 'lead' }, + ], + }, + }) + + Object.assign(process.env, { + GITEA_BASE_URL: gitea.baseUrl, + GITEA_API_TOKEN: 'test-token', + APPWRITE_ENDPOINT: aw.endpoint, + APPWRITE_API_KEY: 'test-key', + PREVIEW_DEPLOY_ROOT: '', + TRAEFIK_DYNAMIC_DIR: '', + }) + // config.js liest die Umgebung beim Import + ;({ config } = await import('../server/config.js')) + ;({ listAllAccessibleRepos, importNewRepos, syncReposToAppwrite, isHostedProject, ensureRepoPreviewConfig } = + await import('../server/services/giteaAdmin.js')) +}) + +after(async () => { + await gitea.close() + await aw.close() +}) + +test('importNewRepos legt nur fehlende an', async () => { + assert.deepEqual(await importNewRepos(), { created: 2 }) + const neu = aw.all('websiteProjects').find((d) => d.repoFullName === 'JUSN/neu') + assert.deepEqual([neu.projectName, neu.status, neu.subdomain || '', neu.previewUrl || ''], ['Döner Nation', 'synced', '', '']) + assert.ok(aw.all('websiteProjects').some((d) => d.repoFullName === 'knso/privat')) + assert.ok(!aw.all('websiteProjects').some((d) => d.repoFullName === 'JUSN/archiv')) + assert.equal(aw.calls.filter((c) => c.method === 'PATCH').length, 0) +}) +test('importNewRepos läuft nie doppelt', async () => { + gitea.addRepo({ full_name: 'ANDJ/zweimal' }) + await Promise.all([importNewRepos(), importNewRepos()]) + assert.equal(aw.all('websiteProjects').filter((d) => d.repoFullName === 'ANDJ/zweimal').length, 1) +}) +test('Abgleich vergibt keine Subdomain mehr', async () => { + gitea.addRepo({ full_name: 'ANDJ/sync-neu' }) + await syncReposToAppwrite() + const d = aw.all('websiteProjects').find((x) => x.repoFullName === 'ANDJ/sync-neu') + assert.equal(d.subdomain || '', ''); assert.equal(d.previewUrl || '', '') + assert.equal(aw.all('websiteProjects').find((x) => x.repoFullName === 'knso/alt').subdomain, 'alt') +}) +test('isHostedProject', () => { + for (const [doc, want] of [[{ projectType: 'preview' }, true], [{ projectType: 'website' }, true], [{ projectType: 'template' }, true], + [{ status: 'deployed' }, true], [{ status: 'pending' }, true], [{ status: 'synced', subdomain: 'x' }, false], [{}, false]]) { + assert.equal(isHostedProject(doc), want, JSON.stringify(doc)) + } +}) +test('Selbstheilung nur für gehostete Projekte', async () => { + assert.equal(await ensureRepoPreviewConfig('knso/synced-sub', 'main'), null) + assert.equal(gitea.calls.filter((c) => c.path.includes('synced-sub/contents')).length, 0) + const cfg = await ensureRepoPreviewConfig('knso/lead', 'main') + assert.equal(cfg.subdomain, 'lead') + assert.ok(gitea.calls.some((c) => c.method === 'POST' && c.path.includes('knso/lead/contents/.webklar-preview.json'))) +}) +test('Repo-Liste hat eine Obergrenze', async () => { + // R8: eigener Fake-Server + kurzzeitig umgebogene config.gitea.baseUrl (statt in `gitea` oben + // Tausende Repos zu seeden) - sonst wuerden importNewRepos/syncReposToAppwrite in den Tests + // darueber auf einmal Tausende zusaetzliche, unbenannte Repos mitverarbeiten. + const capRepos = [] + for (let i = 0; i < 2005; i++) capRepos.push({ full_name: `cap/repo${i}` }) + const capGitea = await startFakeGitea({ repos: capRepos }) + const originalBaseUrl = config.gitea.baseUrl + const originalWarn = console.warn + console.warn = () => {} + config.gitea.baseUrl = capGitea.baseUrl + let repos + try { + repos = await listAllAccessibleRepos() + } finally { + config.gitea.baseUrl = originalBaseUrl + console.warn = originalWarn + await capGitea.close() + } + assert.equal(repos.length, 2000) + assert.equal(capGitea.calls.filter((c) => c.path === '/api/v1/repos/search').length, 40) +})