From 1f1ffb527596ed553fb298b5c896ccc1993b8ab9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 17:36:12 +0000 Subject: [PATCH] Projektarten: neue Repos erkennen und Zuordnungs-Mail senden Co-Authored-By: Claude Sonnet 5.5 --- server/index.js | 9 ++ server/routes/webhook/gitea.js | 15 +++ server/services/giteaAdmin.js | 6 +- server/services/repoAssignJob.js | 139 +++++++++++++++++++++++++++ test/giteaAdmin.test.js | 5 +- test/helpers/fakeGitea.js | 1 + test/repoAssignDetect.test.js | 160 +++++++++++++++++++++++++++++++ 7 files changed, 332 insertions(+), 3 deletions(-) create mode 100644 server/services/repoAssignJob.js create mode 100644 test/repoAssignDetect.test.js diff --git a/server/index.js b/server/index.js index 78c0dce..1394a29 100644 --- a/server/index.js +++ b/server/index.js @@ -1,3 +1,4 @@ +import { runRepoAssignJob } from './services/repoAssignJob.js' import express from 'express' import path from 'node:path' import { fileURLToPath } from 'node:url' @@ -161,6 +162,14 @@ const server = app.listen(config.port, '0.0.0.0', () => { .then((done) => done.length && console.log(`[funktionen] ${done.length} Funktion(en) freigeschaltet`)) .catch((err) => console.warn('[funktionen] Freischalten fehlgeschlagen:', err.message)) setInterval(activateFeatures, 5 * 60 * 1000).unref() + + // Neue Gitea-Repos erkennen, Zuordnungs-Mail senden, Erinnerungen/Wiederholungen (alle 10 Minuten) + const repoAssign = () => + runRepoAssignJob({ now: new Date() }).catch((err) => + console.warn('[repoAssign] Lauf fehlgeschlagen:', err.message) + ) + setTimeout(repoAssign, 60 * 1000).unref() + setInterval(repoAssign, 10 * 60 * 1000).unref() }) server.on('error', (err) => { diff --git a/server/routes/webhook/gitea.js b/server/routes/webhook/gitea.js index 229fa28..6561653 100644 --- a/server/routes/webhook/gitea.js +++ b/server/routes/webhook/gitea.js @@ -15,6 +15,7 @@ import { ensureProjectTicket } from '../../services/ticketAdmin.js' import { invalidateProjectCache } from '../../services/projectInsights.js' import { withLocks } from '../../services/keyedLock.js' import { repoLockKey, currentRepoFullName } from '../../services/repoMove.js' +import { handleRepositoryCreated } from '../../services/repoAssignJob.js' const router = Router() @@ -45,6 +46,20 @@ router.post('/gitea', async (req, res) => { } const payload = req.body || {} + + // Ereignis "repository": nur `created` wird verarbeitet (neues Repo -> Dokument + Zuordnungs-Mail), + // alles andere (deleted, transfer, ...) wird mit 200 ignoriert. Der Push-Pfad bleibt unberuehrt. + if (req.get('X-Gitea-Event') === 'repository') { + if (payload.action !== 'created') return res.json({ ok: true, ignored: true }) + try { + const r = await handleRepositoryCreated(payload) + return res.json({ ok: true, created: r.created }) + } catch (err) { + console.error('[webhook] repository/created:', err) + return res.status(500).json({ error: err.message || 'Webhook-Verarbeitung fehlgeschlagen' }) + } + } + const ref = payload.ref || '' const repo = payload.repository || {} const payloadName = repo.full_name || '' diff --git a/server/services/giteaAdmin.js b/server/services/giteaAdmin.js index 157b5a6..d3f9649 100644 --- a/server/services/giteaAdmin.js +++ b/server/services/giteaAdmin.js @@ -68,6 +68,7 @@ export async function importNewRepos() { ) let created = 0 + const createdRepos = [] for (const repo of repos) { const fullName = repo.full_name if (!fullName || repo.archived) continue @@ -80,7 +81,7 @@ export async function importNewRepos() { ]) if (recheck[0]) return - await upsertWebsiteProjectByRepo(fullName, { + const doc = await upsertWebsiteProjectByRepo(fullName, { projectName: repo.description || repo.name || fullName.split('/').pop(), giteaRepoUrl: repo.html_url, giteaRepoName: repo.name, @@ -90,10 +91,11 @@ export async function importNewRepos() { giteaSyncedAt: new Date().toISOString(), }) created += 1 + createdRepos.push({ projectId: doc.$id, repo }) }) } - return { created } + return { created, createdRepos } }) } diff --git a/server/services/repoAssignJob.js b/server/services/repoAssignJob.js new file mode 100644 index 0000000..3986c26 --- /dev/null +++ b/server/services/repoAssignJob.js @@ -0,0 +1,139 @@ +import { config } from '../config.js' +import { listDocuments, updateDocument, getDocument, upsertWebsiteProjectByRepo, Query } from './appwriteAdmin.js' +import { importNewRepos } from './giteaAdmin.js' +import { giteaFetch } from './giteaClient.js' +import { withLocks } from './keyedLock.js' +import { repoLockKey } from './repoMove.js' +import { shouldSendAssignMail, dueReminder, sendRepoAssignMail } from './repoAssignMail.js' + +/** + * Erkennung neuer Repos (Task 7): der Webhook `repository/created` und der 10-Minuten-Job laufen + * ueber dieselbe Repo-Sperre (repoLockKey) und legen das Projekt-Dokument hoechstens einmal an. + * Die Mail geht ausserhalb dieser Sperre raus, unter einer eigenen Sperre je Projekt, die vor dem + * Senden den Stand frisch liest - so kann kein zweiter Pfad dieselbe Mail doppelt verschicken (R7). + */ + +// Ein vom Job angelegtes Org-Dokument, das der Webhook kurz danach meldet, gilt noch als "neu". +const RACE_WINDOW_MS = 15 * 60 * 1000 + +export const isOrgOwner = (repo) => + String(repo?.owner?.login || '').toLowerCase() === String(config.gitea.org || '').toLowerCase() + +const collection = () => config.collections.websiteProjects + +/** + * Mail senden, falls nach frischem Stand faellig. `createdNow` bedeutet: dies ist der Pfad, der das + * Dokument gerade angelegt hat (oder die Wiederholung einer fehlgeschlagenen ersten Mail). + */ +async function sendFirstMailIfDue(projectId, repo, giteaLogin, { now, createdNow, sendMail }) { + return withLocks(['assignmail:' + projectId], async () => { + const fresh = await getDocument(collection(), projectId) + if (!shouldSendAssignMail(fresh, repo, { now, createdNow })) return { sent: false, reason: 'not_due' } + return sendMail(fresh, { giteaLogin, reminder: false, repoCreatedAt: repo.created_at }) + }) +} + +/** + * Webhook `repository` / `created`: Dokument anlegen (falls fehlt), Ersteller merken, Mail senden. + * Gibt { created, mail } zurueck; `mail` ist das Ergebnis von sendRepoAssignMail oder null. + */ +export async function handleRepositoryCreated(payload, { now = new Date(), sendMail = sendRepoAssignMail } = {}) { + const repo = payload?.repository || {} + const fullName = repo.full_name + const login = payload?.sender?.login || '' + if (!fullName || repo.archived) return { created: false, mail: null } + + let projectId = '' + let created = false + let takeOver = false + await withLocks( + [repoLockKey(fullName)], + async () => { + const existing = ( + await listDocuments(collection(), [Query.equal('repoFullName', fullName), Query.limit(1)]) + )[0] + if (existing) { + projectId = existing.$id + // Vom Job Sekunden zuvor angelegtes Org-Repo ohne Ersteller: der Webhook kennt den Ersteller. + const young = now.getTime() - new Date(existing.$createdAt || 0).getTime() < RACE_WINDOW_MS + if (young && login && !existing.createdByGitea && !existing.kind && !existing.assignMailSentAt && + !existing.customerId && !existing.ticketId && !Number(existing.assignMailAttempts || 0)) { + await updateDocument(collection(), existing.$id, { createdByGitea: login }) + takeOver = true + } + return + } + const doc = await upsertWebsiteProjectByRepo(fullName, { + projectName: repo.description || repo.name || fullName.split('/').pop(), + giteaRepoUrl: repo.html_url || `${config.gitea.baseUrl}/${fullName}`, + giteaRepoName: repo.name || fullName.split('/').pop(), + repoFullName: fullName, + archived: false, + status: 'synced', + giteaSyncedAt: new Date().toISOString(), + createdByGitea: login, + }) + projectId = doc.$id + created = true + }, + { waitMs: 10000 } + ) + + if (!created && !takeOver) return { created, mail: null } + const mail = await sendFirstMailIfDue(projectId, repo, login, { now, createdNow: true, sendMail }) + return { created, mail } +} + +/** Ein Lauf: neue Repos importieren + Mail, Wiederholungen, Erinnerungen. Wirft bei Appwrite-Fehlern nicht. */ +export async function runRepoAssignJob({ + now = new Date(), + sendMail = sendRepoAssignMail, + fetchRepo = (fullName) => giteaFetch(`/repos/${fullName}`), +} = {}) { + const counts = { imported: 0, mails: 0, retries: 0, reminders: 0, error: null } + try { + const { createdRepos = [] } = await importNewRepos() + counts.imported = createdRepos.length + const handled = new Set() // in diesem Lauf schon versucht: nicht sofort wiederholen + + for (const { projectId, repo } of createdRepos) { + // Org-Repos: Ersteller unbekannt, der Webhook-Sender deckt sie ab. + if (isOrgOwner(repo)) continue + const login = repo.owner?.login || '' + if (!login) continue + handled.add(projectId) + await updateDocument(collection(), projectId, { createdByGitea: login }) + const r = await sendFirstMailIfDue(projectId, repo, login, { now, createdNow: true, sendMail }) + if (r.sent) counts.mails += 1 + } + + const docs = await listDocuments(collection(), [Query.limit(5000)]) + for (const p of docs) { + if (!p.createdByGitea || p.kind || handled.has(p.$id)) continue + // Wiederholung: erste Mail fehlgeschlagen (1-2 Versuche), Repo noch juenger als 24 h. + const attempts = Number(p.assignMailAttempts || 0) + if (!p.assignMailSentAt && attempts > 0 && attempts < 3) { + let repo + try { + repo = await fetchRepo(p.repoFullName) + } catch (err) { + console.warn(`[repoAssign] Repo ${p.repoFullName} nicht lesbar:`, err.message) + continue + } + const r = await sendFirstMailIfDue(p.$id, repo, p.createdByGitea, { now, createdNow: true, sendMail }) + if (r.sent) counts.retries += 1 + } else if (dueReminder(p, { now })) { + const r = await withLocks(['assignmail:' + p.$id], async () => { + const fresh = await getDocument(collection(), p.$id) + if (!dueReminder(fresh, { now })) return { sent: false } + return sendMail(fresh, { giteaLogin: fresh.createdByGitea, reminder: true }) + }) + if (r.sent) counts.reminders += 1 + } + } + } catch (err) { + counts.error = err.message + console.error('[repoAssign] Lauf abgebrochen:', err.message) + } + return counts +} diff --git a/test/giteaAdmin.test.js b/test/giteaAdmin.test.js index 8b2c6e3..a5dea40 100644 --- a/test/giteaAdmin.test.js +++ b/test/giteaAdmin.test.js @@ -54,7 +54,10 @@ after(async () => { }) test('importNewRepos legt nur fehlende an', async () => { - assert.deepEqual(await importNewRepos(), { created: 2 }) + const r = await importNewRepos() + assert.equal(r.created, 2) + assert.deepEqual(r.createdRepos.map((x) => x.repo.full_name).sort(), ['JUSN/neu', 'knso/privat']) + assert.ok(r.createdRepos.every((x) => x.projectId)) const neu = aw.all('websiteProjects').find((d) => d.repoFullName === 'JUSN/neu') assert.deepEqual([neu.projectName, neu.status, neu.subdomain || '', neu.previewUrl || ''], ['Döner Nation', 'synced', '', '']) assert.ok(aw.all('websiteProjects').some((d) => d.repoFullName === 'knso/privat')) diff --git a/test/helpers/fakeGitea.js b/test/helpers/fakeGitea.js index 98d4afc..25b9ea7 100644 --- a/test/helpers/fakeGitea.js +++ b/test/helpers/fakeGitea.js @@ -137,6 +137,7 @@ export async function startFakeGitea(options = {}) { private: Boolean(def.private), description: def.description || '', empty: false, + created_at: def.created_at || new Date().toISOString(), } repos.set(keyFrom(fullName), record) return record diff --git a/test/repoAssignDetect.test.js b/test/repoAssignDetect.test.js new file mode 100644 index 0000000..5891910 --- /dev/null +++ b/test/repoAssignDetect.test.js @@ -0,0 +1,160 @@ +import { test, before, after } from 'node:test' +import assert from 'node:assert/strict' +import { createHmac } from 'node:crypto' +import express from 'express' +import { startFakeGitea } from './helpers/fakeGitea.js' +import { startFakeAppwrite } from './helpers/fakeAppwrite.js' +import { mailSink } from './support/mailSink.js' + +const TOKEN = 's3cret' +const NOW = new Date() +const ago = (h) => new Date(NOW.getTime() - h * 3600_000).toISOString() + +let gitea, aw, server, baseUrl, setMailTransportForTests, runRepoAssignJob, sink + +async function hook(event, payload) { + const body = JSON.stringify(payload) + const res = await fetch(`${baseUrl}/webhook/gitea`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Gitea-Event': event, + 'X-Gitea-Signature': createHmac('sha256', TOKEN).update(body).digest('hex'), + }, + body, + }) + return { status: res.status, body: await res.json().catch(() => null) } +} +const created = (full, extra = {}) => ({ + action: 'created', + repository: { full_name: full, name: full.split('/')[1], html_url: `${gitea.baseUrl}/${full}`, created_at: NOW.toISOString(), owner: { login: full.split('/')[0] } }, + sender: { login: 'knso' }, + ...extra, +}) +const docsOf = (full) => aw.all('websiteProjects').filter((d) => d.repoFullName === full) + +before(async () => { + gitea = await startFakeGitea({ repos: [] }) + aw = await startFakeAppwrite({ + seed: { + websiteProjects: [], + employees: [{ $id: 'k', displayName: 'Kenso Grimm', shortcode: 'KNSO', email: 'kenso@webklar.com' }], + }, + }) + Object.assign(process.env, { + GITEA_BASE_URL: gitea.baseUrl, GITEA_API_TOKEN: 'test-token', GITEA_WEBHOOK_TOKEN: TOKEN, + APPWRITE_ENDPOINT: aw.endpoint, APPWRITE_API_KEY: 'test-key', PREVIEW_DEPLOY_ROOT: '', TRAEFIK_DYNAMIC_DIR: '', + }) + const [{ default: routes }, mailer, job] = await Promise.all([ + import('../server/routes/webhook/gitea.js'), + import('../server/services/mailer.js'), + import('../server/services/repoAssignJob.js'), + ]) + setMailTransportForTests = mailer.setMailTransportForTests + runRepoAssignJob = job.runRepoAssignJob + const app = express() + app.use(express.json({ verify: (req, _r, buf) => { req.rawBody = buf } })) + app.use('/webhook', routes) + server = await new Promise((r) => { const s = app.listen(0, '127.0.0.1', () => r(s)) }) + baseUrl = `http://127.0.0.1:${server.address().port}` +}) +after(async () => { + setMailTransportForTests?.(null) + await new Promise((r) => server.close(r)) + await gitea.close() + await aw.close() +}) +const freshSink = () => { sink = mailSink(); setMailTransportForTests(sink) } + +test('repository/created → 1 Dokument + 1 Mail; Wiederholung → keine zweite', async () => { + freshSink() + const r = await hook('repository', created('knso/neu1')) + assert.equal(r.status, 200) + assert.equal(docsOf('knso/neu1').length, 1) + assert.equal(docsOf('knso/neu1')[0].createdByGitea, 'knso') + assert.equal(sink.sent.length, 1) + assert.match(sink.sent[0].html, /ticket\.webklar\.com\/projects\/zuordnen\//) + await hook('repository', created('knso/neu1')) + assert.equal(docsOf('knso/neu1').length, 1) + assert.equal(sink.sent.length, 1) +}) + +test('gleichzeitig Webhook + Job für dasselbe Repo → 1 Dokument, 1 Mail', async () => { + freshSink() + gitea.addRepo({ full_name: 'knso/race', created_at: NOW.toISOString() }) + await Promise.all([hook('repository', created('knso/race')), runRepoAssignJob({ now: NOW }), hook('repository', created('knso/race'))]) + assert.equal(docsOf('knso/race').length, 1) + assert.equal(sink.sent.length, 1) +}) + +test('repository/deleted und ohne Signatur', async () => { + freshSink() + const before = aw.all('websiteProjects').length + const r = await hook('repository', { ...created('knso/weg'), action: 'deleted' }) + assert.equal(r.status, 200) + assert.equal(aw.all('websiteProjects').length, before) + assert.equal(sink.sent.length, 0) + const res = await fetch(`${baseUrl}/webhook/gitea`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Gitea-Event': 'repository' }, body: '{}' }) + assert.equal(res.status, 401) +}) + +test('Job: 20 alte Repos → 0 Mails', async () => { + freshSink() + for (let i = 0; i < 20; i++) gitea.addRepo({ full_name: `knso/alt${i}`, created_at: ago(72) }) + const c = await runRepoAssignJob({ now: NOW }) + assert.equal(c.imported, 20) + assert.equal(c.mails, 0) + assert.equal(sink.sent.length, 0) +}) + +test('Job: neues Nutzer-Repo → Mail, Org-Repo → keine Mail', async () => { + freshSink() + gitea.addRepo({ full_name: 'knso/jobneu', created_at: ago(1) }) + gitea.addRepo({ full_name: 'WEBklar/orgneu', created_at: ago(1) }) + const c = await runRepoAssignJob({ now: NOW }) + assert.equal(c.mails, 1) + assert.equal(sink.sent.length, 1) + assert.equal(docsOf('knso/jobneu')[0].createdByGitea, 'knso') + assert.ok(!docsOf('WEBklar/orgneu')[0].createdByGitea) + assert.equal((await runRepoAssignJob({ now: NOW })).mails, 0) +}) + +test('Job: Erinnerung nach 24 h genau einmal', async () => { + freshSink() + await runRepoAssignJob({ now: NOW }) + assert.equal(sink.sent.length, 0) + const later = new Date(NOW.getTime() + 25 * 3600_000) + assert.equal((await runRepoAssignJob({ now: later })).reminders >= 1, true) + const n = sink.sent.length + assert.ok(n >= 1) + assert.equal((await runRepoAssignJob({ now: later })).reminders, 0) + assert.equal(sink.sent.length, n) +}) + +test('Job: fehlgeschlagene erste Mail wird wiederholt (< 3 Versuche)', async () => { + gitea.addRepo({ full_name: 'knso/retry', created_at: ago(1) }) + setMailTransportForTests({ async sendMail() { throw new Error('smtp down') } }) + const silent = console.error; console.error = () => {} + try { await runRepoAssignJob({ now: NOW }) } finally { console.error = silent } + assert.equal(docsOf('knso/retry')[0].assignMailAttempts, 1) + assert.ok(!docsOf('knso/retry')[0].assignMailSentAt) + freshSink() + const c = await runRepoAssignJob({ now: NOW }) + assert.equal(c.retries, 1) + assert.equal(sink.sent.length, 1) + assert.ok(docsOf('knso/retry')[0].assignMailSentAt) +}) + +test('Job: Appwrite-Fehler bricht den Lauf ab, ohne zu werfen', async () => { + const silent = console.error; console.error = () => {} + const awUrl = process.env.APPWRITE_ENDPOINT + try { + const { config } = await import('../server/config.js') + const old = config.appwrite.endpoint + config.appwrite.endpoint = 'http://127.0.0.1:1/v1' + const c = await runRepoAssignJob({ now: NOW }) + config.appwrite.endpoint = old + assert.ok(c.error) + } finally { console.error = silent } + assert.ok(awUrl) +})