ANDJ/Emailsorter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Try

Browse Source 
dfssdfsfdsf
This commit is contained in:
ANDJ
2026-04-09 21:00:04 +02:00
parent 983b67e6fc
commit 89bc86b615
27 changed files with 2921 additions and 408 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
server/.env
View File

@@ -1,7 +1,7 @@
# Server
PORT=3000
PORT=3030
NODE_ENV=development
BASE_URL=http://localhost:3000
BASE_URL=http://localhost:3030
FRONTEND_URL=http://localhost:5173
# Appwrite (Self-Hosted)
@@ -23,4 +23,4 @@ MISTRAL_API_KEY=yPe00wetm26x9FW4Ifjom2UaEd0hf1ND
# Google OAuth (NEU)
GOOGLE_CLIENT_ID=1073365670500-a6t1srj1ogu1bumoo20511mq4nesouul.apps.googleusercontent.com
GOOGLE_CLIENT_SECRET=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
GOOGLE_REDIRECT_URI=http://localhost:3000/api/oauth/gmail/callback
GOOGLE_REDIRECT_URI=http://localhost:3030/api/oauth/gmail/callback

69
server/config/index.mjs
View File

@@ -3,20 +3,47 @@
* Centralized configuration management
*/
import { readFileSync, existsSync } from 'fs'
import { dirname, join } from 'path'
import { fileURLToPath } from 'url'
import { log } from '../middleware/logger.mjs'
const _configDir = dirname(fileURLToPath(import.meta.url))
const _repoRoot = join(_configDir, '..', '..')
/** Default dev API port from repo root `mailflow.dev.port.json` (avoids clashes with other apps on 3000). */
function readDevPortFromFile() {
try {
const f = join(_repoRoot, 'mailflow.dev.port.json')
if (!existsSync(f)) return 3030
const j = JSON.parse(readFileSync(f, 'utf8'))
const p = parseInt(String(j.port), 10)
return Number.isFinite(p) && p > 0 ? p : 3030
} catch {
return 3030
}
}
const nodeEnv = process.env.NODE_ENV || 'development'
const listenPort = process.env.PORT
? parseInt(process.env.PORT, 10)
: nodeEnv === 'production'
? 3000
: readDevPortFromFile()
const defaultLocalBase = `http://localhost:${listenPort}`
/**
* Environment configuration
*/
export const config = {
// Server
port: parseInt(process.env.PORT || '3000', 10),
nodeEnv: process.env.NODE_ENV || 'development',
isDev: process.env.NODE_ENV !== 'production',
isProd: process.env.NODE_ENV === 'production',
port: listenPort,
nodeEnv,
isDev: nodeEnv !== 'production',
isProd: nodeEnv === 'production',
// URLs
baseUrl: process.env.BASE_URL || 'http://localhost:3000',
baseUrl: process.env.BASE_URL || defaultLocalBase,
frontendUrl: process.env.FRONTEND_URL || 'http://localhost:5173',
// Appwrite
@@ -42,14 +69,14 @@ export const config = {
google: {
clientId: process.env.GOOGLE_CLIENT_ID,
clientSecret: process.env.GOOGLE_CLIENT_SECRET,
redirectUri: process.env.GOOGLE_REDIRECT_URI || 'http://localhost:3000/api/oauth/gmail/callback',
redirectUri: process.env.GOOGLE_REDIRECT_URI || `${defaultLocalBase}/api/oauth/gmail/callback`,
},
// Microsoft OAuth
microsoft: {
clientId: process.env.MICROSOFT_CLIENT_ID,
clientSecret: process.env.MICROSOFT_CLIENT_SECRET,
redirectUri: process.env.MICROSOFT_REDIRECT_URI || 'http://localhost:3000/api/oauth/outlook/callback',
redirectUri: process.env.MICROSOFT_REDIRECT_URI || `${defaultLocalBase}/api/oauth/outlook/callback`,
},
// Mistral AI
@@ -63,9 +90,19 @@ export const config = {
max: parseInt(process.env.RATE_LIMIT_MAX || '100', 10),
},
// CORS
// CORS (Dev: localhost + 127.0.0.1 für Vite, Browser ruft API oft direkt auf 127.0.0.1:PORT)
cors: {
origin: process.env.CORS_ORIGIN || process.env.FRONTEND_URL || 'http://localhost:5173',
origin:
process.env.NODE_ENV === 'production'
? process.env.CORS_ORIGIN || process.env.FRONTEND_URL || 'http://localhost:5173'
: [
process.env.CORS_ORIGIN,
process.env.FRONTEND_URL,
'http://localhost:5173',
'http://127.0.0.1:5173',
]
.filter(Boolean)
.filter((o, i, a) => a.indexOf(o) === i),
credentials: true,
},
@@ -79,11 +116,15 @@ export const config = {
/** Highest product tier (admin comped plan, PLANS key in stripe.mjs). Optional env: TOP_SUBSCRIPTION_PLAN */
topSubscriptionPlan: (process.env.TOP_SUBSCRIPTION_PLAN || 'business').trim().toLowerCase(),
// Admin: comma-separated list of emails with admin rights (e.g. support)
adminEmails: (process.env.ADMIN_EMAILS || '')
.split(',')
.map((e) => e.trim().toLowerCase())
.filter(Boolean),
// Admin: comma-separated list of emails with admin rights (e.g. support).
// support@webklar.com is always included; env adds more.
adminEmails: (() => {
const fromEnv = (process.env.ADMIN_EMAILS || '')
.split(',')
.map((e) => e.trim().toLowerCase())
.filter(Boolean)
return [...new Set(['support@webklar.com', ...fromEnv])]
})(),
// Gitea Webhook (Deployment) — trim: trailing newlines in .env break HMAC/Bearer match
gitea: (() => {

8
server/env.example
View File

@@ -6,9 +6,9 @@
# ─────────────────────────────────────────────────────────────────────────────
# Server Einstellungen
# ─────────────────────────────────────────────────────────────────────────────
PORT=3000
PORT=3030
NODE_ENV=development
BASE_URL=http://localhost:3000
BASE_URL=http://localhost:3030
FRONTEND_URL=http://localhost:5173
# CORS Einstellungen (optional, nutzt FRONTEND_URL als Default)
@@ -51,7 +51,7 @@ MISTRAL_API_KEY=dein_mistral_api_key
# 4. OAuth 2.0 Credentials erstellen
GOOGLE_CLIENT_ID=xxx.apps.googleusercontent.com
GOOGLE_CLIENT_SECRET=GOCSPX-xxx
GOOGLE_REDIRECT_URI=http://localhost:3000/api/oauth/gmail/callback
GOOGLE_REDIRECT_URI=http://localhost:3030/api/oauth/gmail/callback
# ─────────────────────────────────────────────────────────────────────────────
# Microsoft OAuth - Outlook Integration (OPTIONAL)
@@ -63,7 +63,7 @@ GOOGLE_REDIRECT_URI=http://localhost:3000/api/oauth/gmail/callback
# 4. Redirect URI konfigurieren
MICROSOFT_CLIENT_ID=xxx-xxx-xxx
MICROSOFT_CLIENT_SECRET=xxx
MICROSOFT_REDIRECT_URI=http://localhost:3000/api/oauth/outlook/callback
MICROSOFT_REDIRECT_URI=http://localhost:3030/api/oauth/outlook/callback
# ─────────────────────────────────────────────────────────────────────────────
# Admin (OPTIONAL)

59
server/index.mjs
View File

@@ -11,20 +11,24 @@ import { dirname, join } from 'path'
// Config & Middleware
import { config, validateConfig } from './config/index.mjs'
import { errorHandler, asyncHandler, AppError, NotFoundError, ValidationError, AuthorizationError } from './middleware/errorHandler.mjs'
import { errorHandler, asyncHandler, AppError, ValidationError, AuthorizationError } from './middleware/errorHandler.mjs'
import { respond } from './utils/response.mjs'
import { logger, log } from './middleware/logger.mjs'
import { limiters } from './middleware/rateLimit.mjs'
import { requireAuth } from './middleware/auth.mjs'
import { requireAuth, requireAuthUnlessEmailWebhook } from './middleware/auth.mjs'
// Routes
import oauthRoutes from './routes/oauth.mjs'
import emailRoutes from './routes/email.mjs'
import { handleGetDigest } from './routes/email.mjs'
import stripeRoutes from './routes/stripe.mjs'
import { handleGetSubscriptionStatus } from './routes/stripe.mjs'
import apiRoutes from './routes/api.mjs'
import { handleGetReferralCode } from './routes/api.mjs'
import analyticsRoutes from './routes/analytics.mjs'
import webhookRoutes from './routes/webhook.mjs'
import { startCounterJobs } from './jobs/reset-counters.mjs'
import { startAutoSortJob } from './jobs/auto-sort.mjs'
const __filename = fileURLToPath(import.meta.url)
const __dirname = dirname(__filename)
@@ -45,6 +49,21 @@ app.use((req, res, next) => {
next()
})
// Safety net (before all routers): collapse /api/api → /api in path until stable — old clients / bad env
app.use((req, _res, next) => {
const [pathPart, ...q] = req.originalUrl.split('?')
let p = pathPart
let prev = ''
while (p !== prev && p.includes('/api/api')) {
prev = p
p = p.replace(/\/api\/api/g, '/api')
}
if (p !== pathPart) {
req.url = q.length ? `${p}?${q.join('?')}` : p
}
next()
})
// CORS
app.use(cors(config.cors))
@@ -74,7 +93,9 @@ app.get('/api/health', (req, res) => {
res.json({
success: true,
data: {
service: 'mailflow-api',
status: 'healthy',
port: config.port,
timestamp: new Date().toISOString(),
version: process.env.npm_package_version || '1.0.0',
environment: config.nodeEnv,
@@ -83,6 +104,18 @@ app.get('/api/health', (req, res) => {
})
})
/*
* Route index — these three are implemented in routes/*.mjs and registered here FIRST
* (before app.use mounts) so GET always matches real handlers, not the JSON 404 catch-all.
*
* GET /api/email/digest → handleGetDigest (routes/email.mjs)
* GET /api/subscription/status → handleGetSubscriptionStatus (routes/stripe.mjs)
* GET /api/referrals/code → handleGetReferralCode (routes/api.mjs)
*/
app.get('/api/email/digest', requireAuthUnlessEmailWebhook, asyncHandler(handleGetDigest))
app.get('/api/subscription/status', requireAuth, asyncHandler(handleGetSubscriptionStatus))
app.get('/api/referrals/code', requireAuth, asyncHandler(handleGetReferralCode))
// API Routes
app.use('/api/oauth', oauthRoutes)
app.use('/api/email', emailRoutes)
@@ -302,20 +335,12 @@ app.delete('/api/preferences/name-labels/:id', requireAuth, asyncHandler(async (
// Legacy Stripe webhook endpoint
app.use('/stripe', stripeRoutes)
// Unmatched /api → JSON 404 (Express 4 treats '/api/*' as a literal path, not a wildcard)
app.use((req, res, next) => {
const pathOnly = req.originalUrl.split('?')[0]
if (!pathOnly.startsWith('/api')) {
return next()
}
next(new NotFoundError('Endpoint'))
})
// SPA fallback: never send index.html for /api (avoids 404/HTML when public/index.html is missing)
app.get('*', (req, res, next) => {
const pathOnly = req.originalUrl.split('?')[0]
if (pathOnly.startsWith('/api')) {
return next(new NotFoundError('Endpoint'))
console.warn('[404] Unmatched route:', req.method, req.originalUrl)
return res.status(404).json({ error: 'Endpoint not found', path: req.originalUrl })
}
const indexPath = join(__dirname, '..', 'public', 'index.html')
res.sendFile(indexPath, (err) => {
@@ -331,6 +356,15 @@ app.get('*', (req, res, next) => {
})
})
// Catch-all: any method/path that did not send a response (e.g. POST /unknown)
app.use((req, res, next) => {
if (res.headersSent) {
return next()
}
console.warn('[404] Unmatched route:', req.method, req.originalUrl)
res.status(404).json({ error: 'Endpoint not found', path: req.originalUrl })
})
// Global error handler (must be last)
app.use(errorHandler)
@@ -376,6 +410,7 @@ server = app.listen(config.port, () => {
console.log(` 💚 Health: http://localhost:${config.port}/api/health`)
console.log('')
startCounterJobs()
startAutoSortJob()
})
export default app

161
server/routes/api.mjs
View File

@@ -4,13 +4,34 @@
*/
import express from 'express'
import { asyncHandler, NotFoundError, ValidationError } from '../middleware/errorHandler.mjs'
import { Client, Users, Query as AppwriteQuery } from 'node-appwrite'
import { asyncHandler, NotFoundError, ValidationError, AuthorizationError } from '../middleware/errorHandler.mjs'
import { validate, schemas, rules } from '../middleware/validate.mjs'
import { respond } from '../utils/response.mjs'
import { products, questions, submissions, orders, onboardingState, emailAccounts, emailStats, emailDigests, userPreferences, subscriptions, emailUsage, referrals, db, Collections, Query } from '../services/database.mjs'
import {
products,
questions,
submissions,
orders,
onboardingState,
emailAccounts,
emailStats,
emailDigests,
userPreferences,
subscriptions,
emailUsage,
referrals,
db,
Collections,
Query,
deleteAllDocumentsForUser,
} from '../services/database.mjs'
import Stripe from 'stripe'
import { config } from '../config/index.mjs'
import { config, isAdmin } from '../config/index.mjs'
import { log } from '../middleware/logger.mjs'
import { parseImapAccountAccess } from './email.mjs'
import { ImapService } from '../services/imap.mjs'
import { isAppwriteCollectionMissing } from '../utils/appwriteErrors.mjs'
import { requireAuth } from '../middleware/auth.mjs'
const router = express.Router()
@@ -328,18 +349,31 @@ router.delete('/account/delete',
/**
* GET /api/referrals/code
* Get or create referral code for user
* (Also registered on app in index.mjs before router mount.)
*/
router.get('/referrals/code',
requireAuth,
asyncHandler(async (req, res) => {
const userId = req.appwriteUser.id
const referral = await referrals.getOrCreateCode(userId)
respond.success(res, {
referralCode: referral.referralCode,
referralCount: referral.referralCount || 0,
export async function handleGetReferralCode(req, res) {
const userId = req.appwriteUser.id
try {
const result = await referrals.getOrCreateCode(userId)
if (!result) {
return respond.success(res, { referralCode: null, referralCount: 0 })
}
return respond.success(res, {
referralCode: result.referralCode,
referralCount: result.referralCount || 0,
})
})
)
} catch (err) {
if (isAppwriteCollectionMissing(err)) {
return respond.success(res, {
referralCode: null,
referralCount: 0,
})
}
throw err
}
}
router.get('/referrals/code', requireAuth, asyncHandler(handleGetReferralCode))
/**
* POST /api/referrals/track
@@ -383,4 +417,105 @@ router.post('/referrals/track',
})
)
async function resolveUserIdByEmail(email) {
const normalized = String(email).trim().toLowerCase()
const c = new Client()
.setEndpoint(config.appwrite.endpoint)
.setProject(config.appwrite.projectId)
.setKey(config.appwrite.apiKey)
const users = new Users(c)
try {
const res = await users.list([AppwriteQuery.equal('email', normalized)])
const uid = res.users?.[0]?.$id
if (uid) return uid
} catch (e) {
log.warn('resolveUserIdByEmail: Users.list failed', { email: normalized, error: e.message })
}
const acc = await db.findOne(Collections.EMAIL_ACCOUNTS, [Query.equal('email', normalized)])
return acc?.userId || null
}
/**
* POST /api/admin/reset-user-sort-data
* Admin only: clear sort-related data for a user (by email).
*/
router.post(
'/admin/reset-user-sort-data',
requireAuth,
validate({
body: {
email: [rules.required('email'), rules.email()],
},
}),
asyncHandler(async (req, res) => {
if (!isAdmin(req.appwriteUser?.email)) {
throw new AuthorizationError('Admin access required')
}
const targetEmail = String(req.body.email).trim().toLowerCase()
const targetUserId = await resolveUserIdByEmail(targetEmail)
if (!targetUserId) {
throw new NotFoundError('User for this email')
}
const statsDoc = await emailStats.getByUser(targetUserId)
let statsDeleted = 0
if (statsDoc?.$id) {
await db.delete(Collections.EMAIL_STATS, statsDoc.$id)
statsDeleted = 1
}
const digestsDeleted = await deleteAllDocumentsForUser(Collections.EMAIL_DIGESTS, targetUserId)
const usageDeleted = await deleteAllDocumentsForUser(Collections.EMAIL_USAGE, targetUserId)
await onboardingState.resetToInitial(targetUserId)
let imapCleared = 0
const imapAccounts = await db.list(Collections.EMAIL_ACCOUNTS, [
Query.equal('userId', targetUserId),
Query.equal('provider', 'imap'),
])
for (const acc of imapAccounts) {
if (!acc.accessToken) continue
try {
const cfg = parseImapAccountAccess(acc)
if (!cfg.password) continue
const imap = new ImapService({
host: cfg.host,
port: cfg.port,
secure: cfg.secure,
user: acc.email,
password: cfg.password,
})
const n = await imap.removeAllSortedFlags()
imapCleared += n
} catch (e) {
log.warn('reset-user-sort-data: IMAP flags failed for account', {
accountId: acc.$id,
error: e.message,
})
}
}
log.info('Admin reset-user-sort-data', {
admin: req.appwriteUser.email,
targetEmail,
targetUserId,
statsDeleted,
digestsDeleted,
usageDeleted,
imapCleared,
})
respond.success(res, {
reset: true,
deleted: {
stats: statsDeleted,
digests: digestsDeleted,
usage: usageDeleted,
},
imapCleared,
})
})
)
export default router

680
server/routes/email.mjs
View File

@@ -13,6 +13,9 @@ import { config, features, isAdmin } from '../config/index.mjs'
import { log } from '../middleware/logger.mjs'
import { requireAuthUnlessEmailWebhook } from '../middleware/auth.mjs'
import { encryptImapSecret, decryptImapSecret } from '../utils/crypto.mjs'
import { isAppwriteCollectionMissing } from '../utils/appwriteErrors.mjs'
import { AI_BATCH_CHUNK_SIZE, AI_BATCH_CHUNK_DELAY_MS } from '../services/ai-sorter.mjs'
import { CATEGORY_FOLDER_KEYWORDS, findBestFolder, findPersonFolder } from '../services/imap.mjs'
const router = express.Router()
@@ -47,6 +50,49 @@ async function getAISorter() {
return aiSorterInstance
}
/** Reject after `ms` so the IMAP sort handler cannot hang indefinitely. */
function imapSortRaceWithTimeout(promise, ms, message) {
let timer
const timeout = new Promise((_, reject) => {
timer = setTimeout(() => reject(new ValidationError(message)), ms)
})
return Promise.race([
promise.finally(() => clearTimeout(timer)),
timeout,
])
}
/**
* IMAP credentials: new accounts store JSON in accessToken; legacy uses encrypted password only.
*/
export function parseImapAccountAccess(account) {
const fallbackHost = account.imapHost || 'imap.porkbun.com'
const fallbackPort = account.imapPort != null ? Number(account.imapPort) : 993
const fallbackSecure = account.imapSecure !== false
if (!account.accessToken) {
return { host: fallbackHost, port: fallbackPort, secure: fallbackSecure, password: '' }
}
try {
const parsed = JSON.parse(account.accessToken)
if (parsed && typeof parsed === 'object' && typeof parsed.password === 'string') {
return {
host: parsed.host ?? fallbackHost,
port: parsed.port != null ? Number(parsed.port) : fallbackPort,
secure: parsed.secure !== false,
password: decryptImapSecret(parsed.password),
}
}
} catch {
// legacy: entire accessToken is encrypted secret
}
return {
host: fallbackHost,
port: fallbackPort,
secure: fallbackSecure,
password: decryptImapSecret(account.accessToken),
}
}
// ═══════════════════════════════════════════════════════════════════════════
// DEMO DATA - Realistic Test Emails
// ═══════════════════════════════════════════════════════════════════════════
@@ -128,23 +174,26 @@ router.post('/connect',
}
}
// Create account
// Create account (IMAP: encode host/port/secure inside accessToken JSON — no Appwrite attrs)
const rawImapSecret = provider === 'imap' ? (password || accessToken) : ''
const accountData = {
userId,
provider,
email,
accessToken: provider === 'imap' ? encryptImapSecret(rawImapSecret) : (accessToken || ''),
accessToken:
provider === 'imap'
? JSON.stringify({
password: encryptImapSecret(rawImapSecret),
host: imapHost || 'imap.porkbun.com',
port: imapPort != null ? Number(imapPort) : 993,
secure: imapSecure !== false,
})
: (accessToken || ''),
refreshToken: provider === 'imap' ? '' : (refreshToken || ''),
expiresAt: provider === 'imap' ? 0 : (expiresAt || 0),
isActive: true,
lastSync: null,
}
if (provider === 'imap') {
if (imapHost != null) accountData.imapHost = String(imapHost)
if (imapPort != null) accountData.imapPort = Number(imapPort)
if (imapSecure !== undefined) accountData.imapSecure = Boolean(imapSecure)
}
const account = await emailAccounts.create(accountData)
log.success(`Email account connected: ${email} (${provider})`)
@@ -280,7 +329,7 @@ router.get('/stats', asyncHandler(async (req, res) => {
* Trigger email sorting for an account
*
* Options:
* - maxEmails: Maximum emails to process (default: 500, max: 2000)
* - maxEmails: Maximum emails to process (default: 500; IMAP default/cap 50, max 2000 non-IMAP)
* - processAll: If true, process entire inbox with pagination
*/
router.post('/sort',
@@ -292,13 +341,33 @@ router.post('/sort',
}),
asyncHandler(async (req, res) => {
const userId = req.appwriteUser.id
const { accountId, maxEmails = 500, processAll = true } = req.body
// Check subscription status and free tier limits
const subscription = await subscriptions.getByUser(userId, req.appwriteUser?.email)
const { accountId, processAll = true } = req.body
console.log('[SORT] Step 1: Auth OK, userId:', userId)
const subscriptionFreeTierDefaults = () => ({
plan: 'free',
status: 'active',
isFreeTier: true,
emailsUsedThisMonth: 0,
emailsLimit: config.freeTier.emailsPerMonth,
})
// Check subscription status and free tier limits (missing collections → treat as free tier)
let subscription
try {
subscription = await subscriptions.getByUser(userId, req.appwriteUser?.email)
} catch {
subscription = null
}
if (!subscription) {
subscription = subscriptionFreeTierDefaults()
}
const isFreeTier = subscription?.isFreeTier || false
const adminUser = isAdmin(req.appwriteUser?.email)
console.log('[SORT] Step 3: Subscription:', subscription?.plan, 'isFreeTier:', subscription?.isFreeTier)
// Check free tier limit (admins: unlimited)
if (isFreeTier && !adminUser) {
const usage = await emailUsage.getUsage(userId)
@@ -314,22 +383,40 @@ router.post('/sort',
}
}
// Check if this is first run (no stats exist)
const existingStats = await emailStats.getByUser(userId)
const isFirstRun = !existingStats || existingStats.totalSorted === 0
// For first run, limit to 50 emails for speed
const effectiveMax = isFirstRun
? Math.min(maxEmails, 50)
: Math.min(maxEmails, 2000) // Cap at 2000 emails
// Get account
const account = await emailAccounts.get(accountId)
if (!account) {
throw new NotFoundError('Email account')
}
if (account.userId !== userId) {
throw new AuthorizationError('No permission for this account')
}
let maxEmails = req.body.maxEmails
if (maxEmails == null) {
maxEmails = 500
} else {
maxEmails = Number(maxEmails)
if (!Number.isFinite(maxEmails) || maxEmails < 0) {
maxEmails = 500
}
}
if (account.provider === 'imap') {
maxEmails = Math.min(maxEmails, 500)
}
const effectiveMax =
account.provider === 'imap'
? Math.min(maxEmails, 500)
: isFirstRun
? Math.min(maxEmails, 50)
: Math.min(maxEmails, 2000)
console.log('[SORT] Step 2: Account fetched:', account?.$id ?? 'NULL', 'provider:', account?.provider)
// Get user preferences
const prefs = await userPreferences.getByUser(userId)
const preferences = prefs?.preferences || {}
@@ -337,6 +424,7 @@ router.post('/sort',
// Get AI sorter
const sorter = await getAISorter()
let sortedCount = 0
let timedOut = false
const results = { byCategory: {} }
let emailSamples = [] // For suggested rules generation
@@ -351,11 +439,15 @@ router.post('/sort',
const shuffled = [...DEMO_EMAILS].sort(() => Math.random() - 0.5)
const emailsToSort = shuffled.slice(0, emailCount)
console.log('[SORT] Step 4: Emails fetched (demo):', emailsToSort?.length ?? 0)
// Check if AI is available
if (features.ai()) {
// Real AI sorting with demo data
const classified = await sorter.batchCategorize(emailsToSort, preferences)
console.log('[SORT] Step 5: Categorized (demo AI):', classified?.length ?? 0)
for (const { email, classification } of classified) {
const category = classification.category
sortedCount++
@@ -427,6 +519,8 @@ router.post('/sort',
log.success(`Rule-based sorting completed: ${sortedCount} demo emails`)
}
console.log('[SORT] Step 5: Categorized (demo final sortedCount):', sortedCount)
}
// ═══════════════════════════════════════════════════════════════════════
// GMAIL - Real Gmail sorting with native categories
@@ -548,6 +642,8 @@ router.post('/sort',
// Get full email details
const emails = await gmail.batchGetEmails(messages.map(m => m.id))
console.log('[SORT] Step 4: Emails fetched (gmail batch):', emails?.length ?? 0)
// Process each email: check company labels first, then AI categorization
const processedEmails = []
@@ -693,6 +789,8 @@ router.post('/sort',
}
} while (pageToken && processAll)
console.log('[SORT] Step 5: Categorized (gmail sortedCount):', sortedCount)
log.success(`Gmail sorting completed: ${sortedCount} emails processed`)
} catch (err) {
log.error('Gmail sorting failed', { error: err.message })
@@ -728,6 +826,8 @@ router.post('/sort',
if (!messages?.length) break
console.log('[SORT] Step 4: Emails fetched (outlook batch):', messages?.length ?? 0)
// Process each email: check company labels first, then AI categorization
const enabledCategories = sorter.getEnabledCategories(preferences)
const processedEmails = []
@@ -785,6 +885,8 @@ router.post('/sort',
})
}
console.log('[SORT] Step 5: Categorized (outlook batch processed):', processedEmails.length)
// Apply categories and actions
for (const { email, category, companyLabel } of processedEmails) {
const action = sorter.getCategoryAction(category, preferences)
@@ -838,6 +940,8 @@ router.post('/sort',
}
} while (skipToken && processAll)
console.log('[SORT] Step 5: Categorized (outlook sortedCount):', sortedCount)
log.success(`Outlook sorting completed: ${sortedCount} emails processed`)
} catch (err) {
log.error('Outlook sorting failed', { error: err.message })
@@ -858,147 +962,272 @@ router.post('/sort',
log.info(`IMAP sorting started for ${account.email}`)
const { ImapService, getFolderNameForCategory } = await import('../services/imap.mjs')
const { ImapService } = await import('../services/imap.mjs')
const imapCfg = parseImapAccountAccess(account)
const imap = new ImapService({
host: account.imapHost || 'imap.porkbun.com',
port: account.imapPort != null ? account.imapPort : 993,
secure: account.imapSecure !== false,
host: imapCfg.host,
port: imapCfg.port,
secure: imapCfg.secure,
user: account.email,
password: decryptImapSecret(account.accessToken),
password: imapCfg.password,
})
try {
await imap.connect()
const enabledCategories = sorter.getEnabledCategories(preferences)
// Name labels (workers): create Team subfolders for IMAP/Nextcloud
const nameLabelMap = {}
if (preferences.nameLabels?.length) {
for (const nl of preferences.nameLabels) {
if (!nl.enabled) continue
const folderName = `Team/${nl.name}`
try {
await imapSortRaceWithTimeout(
(async () => {
try {
await imap.ensureFolder(folderName)
nameLabelMap[nl.id || nl.name] = folderName
if (nl.name !== (nl.id || nl.name)) nameLabelMap[nl.name] = folderName
await imap.connect()
const enabledCategories = sorter.getEnabledCategories(preferences)
const existingFolders = await imap.listAllFolders()
console.log('[SORT-IMAP] All available folders:', existingFolders)
const folderPathSet = new Set(existingFolders)
console.log(`[SORT-IMAP] Folders discovered: ${existingFolders.length}`)
const folderMap = {}
for (const cat of Object.keys(CATEGORY_FOLDER_KEYWORDS)) {
folderMap[cat] = findBestFolder(cat, existingFolders)
}
console.log('[SORT-IMAP] Folder map:', JSON.stringify(folderMap))
const fetchCap = Math.min(500, effectiveMax)
const { messages } = await imap.listEmails(fetchCap, null)
if (!messages?.length) {
console.log('[SORT-IMAP] No messages in INBOX to process')
log.success('IMAP sorting completed: 0 emails processed')
return
}
const emails = await imap.batchGetEmails(messages.map((m) => m.id))
console.log('[SORT-IMAP] Emails fetched (batch):', emails?.length ?? 0)
const processedEmails = []
for (let chunkStart = 0; chunkStart < emails.length; chunkStart += AI_BATCH_CHUNK_SIZE) {
const chunk = emails.slice(chunkStart, chunkStart + AI_BATCH_CHUNK_SIZE)
for (const email of chunk) {
const emailData = {
from: email.headers?.from || '',
subject: email.headers?.subject || '',
snippet: email.snippet || '',
}
let category = null
let companyLabel = null
let assignedTo = null
let skipAI = false
if (preferences.companyLabels?.length) {
for (const companyLabelConfig of preferences.companyLabels) {
if (!companyLabelConfig.enabled) continue
if (sorter.matchesCompanyLabel(emailData, companyLabelConfig)) {
category = companyLabelConfig.category || 'promotions'
companyLabel = companyLabelConfig.name
skipAI = true
break
}
}
}
if (!skipAI && preferences.autoDetectCompanies) {
const detected = sorter.detectCompany(emailData)
if (detected) {
category = 'promotions'
companyLabel = detected.label
skipAI = true
}
}
if (!skipAI) {
const classification = await sorter.categorize(emailData, preferences)
category = classification.category
assignedTo = classification.assignedTo || null
if (!enabledCategories.includes(category)) category = 'review'
}
processedEmails.push({ email, category, companyLabel, assignedTo })
if (isFirstRun && emailSamples.length < 50) {
emailSamples.push({
from: emailData.from,
subject: emailData.subject,
snippet: emailData.snippet,
category,
})
}
}
if (chunkStart + AI_BATCH_CHUNK_SIZE < emails.length) {
await new Promise((r) => setTimeout(r, AI_BATCH_CHUNK_DELAY_MS))
}
}
console.log('[SORT-IMAP] Categorized:', processedEmails.length)
const MOVE_OUT_COMPLETELY = ['newsletters', 'promotions', 'social']
let movedCount = 0
let copiedCount = 0
let sortDecisionLogCount = 0
for (const { email, category, companyLabel, assignedTo } of processedEmails) {
const resolvedCategory = companyLabel
? preferences.companyLabels?.find((c) => c.name === companyLabel)?.category || 'promotions'
: category
const emailDataForPerson = {
from: email.headers?.from || '',
subject: email.headers?.subject || '',
}
const personFolder = findPersonFolder(emailDataForPerson, existingFolders)
const categoryFolder = folderMap[resolvedCategory] || null
const action = sorter.getCategoryAction
? sorter.getCategoryAction(resolvedCategory, preferences)
: 'inbox'
const didEarlyArchiveRead =
MOVE_OUT_COMPLETELY.includes(resolvedCategory) &&
categoryFolder &&
action === 'archive_read'
if (didEarlyArchiveRead) {
try {
await imap.markAsRead(email.id)
} catch {
// ignore — message may already be moved in edge cases
}
}
if (personFolder) {
try {
const copied = await imap.copyToFolder(email.id, personFolder, folderPathSet)
if (copied) {
copiedCount++
console.log(
`[SORT-IMAP] Email ${email.id} → copied to person folder "${personFolder}"`
)
}
} catch {
// ignore
}
}
if (categoryFolder && categoryFolder !== personFolder) {
if (MOVE_OUT_COMPLETELY.includes(resolvedCategory)) {
try {
const moved = await imap.moveMessageToExistingPath(
email.id,
categoryFolder,
folderPathSet
)
if (moved) {
movedCount++
console.log(`[SORT-IMAP] Email ${email.id} → MOVED to "${categoryFolder}"`)
} else {
const copied = await imap.copyToFolder(
email.id,
categoryFolder,
folderPathSet
)
if (copied) {
copiedCount++
console.log(
`[SORT-IMAP] Email ${email.id} → copied (move failed) to "${categoryFolder}"`
)
}
}
} catch {
// ignore
}
} else {
try {
const copied = await imap.copyToFolder(
email.id,
categoryFolder,
folderPathSet
)
if (copied) {
copiedCount++
console.log(
`[SORT-IMAP] Email ${email.id} → copied to "${categoryFolder}"`
)
}
} catch {
// ignore
}
}
}
if (!MOVE_OUT_COMPLETELY.includes(resolvedCategory) || personFolder) {
try {
await imap.addMailFlowCategoryKeyword(
email.id,
resolvedCategory,
assignedTo || null
)
} catch {
// ignore
}
}
try {
if (action === 'archive_read' && !didEarlyArchiveRead) {
await imap.markAsRead(email.id)
}
} catch {
// ignore — e.g. message already moved out of INBOX
}
sortedCount++
results.byCategory[resolvedCategory] = (results.byCategory[resolvedCategory] || 0) + 1
if (sortDecisionLogCount < 20) {
sortDecisionLogCount++
console.log(
`[SORT-IMAP] Decision ${sortDecisionLogCount}/20: uid=${email.id} category=${resolvedCategory} ` +
`personFolder=${personFolder || '—'} categoryFolder=${categoryFolder || '—'} ` +
`moveOut=${MOVE_OUT_COMPLETELY.includes(resolvedCategory)}`
)
}
}
console.log(
`[SORT-IMAP] Complete: ${movedCount} moved, ${copiedCount} copied`
)
log.success(`IMAP sorting completed: ${sortedCount} emails processed`)
} catch (err) {
log.warn(`IMAP name label folder failed: ${nl.name}`, { error: err.message })
log.error('IMAP sorting failed', { error: err.message })
throw new ValidationError(`IMAP error: ${err.message}. Check credentials or reconnect.`)
}
})(),
300_000,
'IMAP sort timed out after 300 seconds. Check host, port, or network.'
)
} catch (err) {
if (String(err?.message || '').includes('timed out')) {
timedOut = true
log.warn(
`[SORT-IMAP] Timed out after 300s — saving partial results (${sortedCount} sorted so far)`
)
} else {
throw err
}
}
let pageToken = null
let totalProcessed = 0
const batchSize = 100
do {
const { messages, nextPageToken } = await imap.listEmails(batchSize, pageToken)
pageToken = nextPageToken
if (!messages?.length) break
const emails = await imap.batchGetEmails(messages.map((m) => m.id))
const processedEmails = []
for (const email of emails) {
const emailData = {
from: email.headers?.from || '',
subject: email.headers?.subject || '',
snippet: email.snippet || '',
}
let category = null
let companyLabel = null
let assignedTo = null
let skipAI = false
if (preferences.companyLabels?.length) {
for (const companyLabelConfig of preferences.companyLabels) {
if (!companyLabelConfig.enabled) continue
if (sorter.matchesCompanyLabel(emailData, companyLabelConfig)) {
category = companyLabelConfig.category || 'promotions'
companyLabel = companyLabelConfig.name
skipAI = true
break
}
}
}
if (!skipAI && preferences.autoDetectCompanies) {
const detected = sorter.detectCompany(emailData)
if (detected) {
category = 'promotions'
companyLabel = detected.label
skipAI = true
}
}
if (!skipAI) {
const classification = await sorter.categorize(emailData, preferences)
category = classification.category
assignedTo = classification.assignedTo || null
if (!enabledCategories.includes(category)) category = 'review'
}
processedEmails.push({ email, category, companyLabel, assignedTo })
if (isFirstRun && emailSamples.length < 50) {
emailSamples.push({
from: emailData.from,
subject: emailData.subject,
snippet: emailData.snippet,
category,
})
}
}
const actionMap = sorter.getCategoryAction ? (cat) => sorter.getCategoryAction(cat, preferences) : () => 'inbox'
for (const { email, category, companyLabel, assignedTo } of processedEmails) {
try {
const action = actionMap(category)
// If AI assigned to a worker, move to Team/<Name> folder; else use category folder
const folderName = (assignedTo && nameLabelMap[assignedTo])
? nameLabelMap[assignedTo]
: getFolderNameForCategory(companyLabel ? (preferences.companyLabels?.find((c) => c.name === companyLabel)?.category || 'promotions') : category)
await imap.moveToFolder(email.id, folderName)
if (action === 'archive_read') {
try {
await imap.markAsRead(email.id)
} catch {
// already moved; mark as read optional
}
}
sortedCount++
results.byCategory[category] = (results.byCategory[category] || 0) + 1
} catch (err) {
log.warn(`IMAP sort failed: ${email.id}`, { error: err.message })
}
}
totalProcessed += emails.length
log.info(`IMAP processed ${totalProcessed} emails so far...`)
if (totalProcessed >= effectiveMax) break
if (pageToken) await new Promise((r) => setTimeout(r, 200))
} while (pageToken && processAll)
await imap.close()
log.success(`IMAP sorting completed: ${sortedCount} emails processed`)
} catch (err) {
} finally {
try {
await imap.close()
} catch {
// ignore
// ignore — dead connection must not block HTTP response
}
log.error('IMAP sorting failed', { error: err.message })
throw new ValidationError(`IMAP error: ${err.message}. Check credentials or reconnect.`)
}
}
console.log('[SORT] Step 6: Saving results (sync, stats, usage, digest)...')
// Update last sync
await emailAccounts.updateLastSync(accountId)
@@ -1075,6 +1304,8 @@ router.post('/sort',
log.warn('Digest update failed', { error: err.message })
}
console.log('[SORT] Step 6: Results saved, sortedCount:', sortedCount)
log.success(`Sorting completed: ${sortedCount} emails for ${account.email}`)
// Generate suggested rules for first run
@@ -1102,10 +1333,122 @@ router.post('/sort',
suggestions,
provider: account.provider,
isDemo: account.provider === 'demo',
timedOut: timedOut || undefined,
message: timedOut
? `Sorted ${sortedCount} emails (sort timed out, will continue next run)`
: undefined,
})
})
)
/**
* POST /api/email/recover/:accountId
* Move messages from all non-INBOX folders back to INBOX (IMAP only).
*/
router.post(
'/recover/:accountId',
asyncHandler(async (req, res) => {
const userId = req.appwriteUser.id
const { accountId } = req.params
const account = await emailAccounts.get(accountId)
if (!account) throw new NotFoundError('Email account')
if (account.userId !== userId) throw new AuthorizationError('No permission for this account')
if (account.provider !== 'imap') {
return respond.success(res, {
recovered: 0,
folders: [],
message: 'Only available for IMAP accounts',
})
}
const { ImapService } = await import('../services/imap.mjs')
const imapCfg = parseImapAccountAccess(account)
const imap = new ImapService({
host: imapCfg.host,
port: imapCfg.port,
secure: imapCfg.secure,
user: account.email,
password: imapCfg.password,
})
log.info(`Email recovery started for ${account.email}`)
const result = await imap.recoverAllToInbox()
log.success(`Recovery complete: ${result.recovered} emails returned to INBOX`)
respond.success(res, {
recovered: result.recovered,
folders: result.folders,
message:
result.recovered > 0
? `${result.recovered} emails recovered back to inbox`
: 'No emails found outside inbox',
})
})
)
/**
* POST /api/email/re-sort/:accountId
* IMAP: move messages from sort-related folders (Junk, Archive, MailFlow/*, …) back to INBOX and strip $MailFlow-* keywords.
*/
router.post(
'/re-sort/:accountId',
asyncHandler(async (req, res) => {
const userId = req.appwriteUser.id
const { accountId } = req.params
const account = await emailAccounts.get(accountId)
if (!account) throw new NotFoundError('Email account')
if (account.userId !== userId) throw new AuthorizationError('No permission for this account')
if (account.provider !== 'imap') {
return respond.success(res, {
recovered: 0,
folders: [],
mailFlowKeywordsStripped: 0,
message: 'Only available for IMAP accounts',
})
}
const { ImapService } = await import('../services/imap.mjs')
const imapCfg = parseImapAccountAccess(account)
const imap = new ImapService({
host: imapCfg.host,
port: imapCfg.port,
secure: imapCfg.secure,
user: account.email,
password: imapCfg.password,
})
log.info(`IMAP re-sort prep started for ${account.email}`)
try {
await imap.connect()
const result = await imap.reSortRecoverAndStripKeywords()
log.success(
`Re-sort prep: ${result.recovered} to INBOX, MailFlow keywords stripped on ${result.mailFlowKeywordsStripped} message(s)`
)
respond.success(res, {
recovered: result.recovered,
folders: result.folders,
mailFlowKeywordsStripped: result.mailFlowKeywordsStripped,
message:
result.recovered > 0 || result.mailFlowKeywordsStripped > 0
? `Moved ${result.recovered} message(s) to INBOX; stripped MailFlow tags from ${result.mailFlowKeywordsStripped} INBOX message(s). Run sort again.`
: 'Nothing to reset — INBOX already clean of sort folders / keywords.',
})
} finally {
try {
await imap.close()
} catch {
/* ignore */
}
}
})
)
/**
* POST /api/email/sort-demo
* Quick demo sorting without account (for testing)
@@ -1233,16 +1576,13 @@ router.post('/cleanup/mailflow-labels',
/**
* GET /api/email/digest
* Get today's sorting digest summary
* (Also registered on app in index.mjs before router mount.)
*/
router.get('/digest', asyncHandler(async (req, res) => {
export async function handleGetDigest(req, res) {
const userId = req.appwriteUser.id
const digest = await emailDigests.getByUserToday(userId)
if (!digest) {
// Return empty digest for new users
return respond.success(res, {
date: new Date().toISOString().split('T')[0],
const emptyDigest = () =>
respond.success(res, {
date: new Date().toISOString(),
totalSorted: 0,
inboxCleared: 0,
timeSavedMinutes: 0,
@@ -1251,19 +1591,33 @@ router.get('/digest', asyncHandler(async (req, res) => {
suggestions: [],
hasData: false,
})
}
respond.success(res, {
date: digest.date,
totalSorted: digest.totalSorted,
inboxCleared: digest.inboxCleared,
timeSavedMinutes: digest.timeSavedMinutes,
stats: digest.stats,
highlights: digest.highlights,
suggestions: digest.suggestions,
hasData: true,
})
}))
try {
const digest = await emailDigests.getByUserToday(userId)
if (!digest) {
return emptyDigest()
}
return respond.success(res, {
date: digest.date,
totalSorted: digest.totalSorted,
inboxCleared: digest.inboxCleared,
timeSavedMinutes: digest.timeSavedMinutes,
stats: digest.stats,
highlights: digest.highlights,
suggestions: digest.suggestions,
hasData: true,
})
} catch (err) {
if (isAppwriteCollectionMissing(err)) {
return emptyDigest()
}
throw err
}
}
router.get('/digest', asyncHandler(handleGetDigest))
/**
* GET /api/email/digest/history

104
server/routes/stripe.mjs
View File

@@ -11,10 +11,11 @@ import { validate, rules } from '../middleware/validate.mjs'
import { limiters } from '../middleware/rateLimit.mjs'
import { respond } from '../utils/response.mjs'
import { subscriptions, submissions } from '../services/database.mjs'
import { config } from '../config/index.mjs'
import { config, isAdmin } from '../config/index.mjs'
import { log } from '../middleware/logger.mjs'
import { requireAuth } from '../middleware/auth.mjs'
import { loadEmailTemplate, renderTemplate, sendPlainEmail } from '../utils/mailer.mjs'
import { isAppwriteCollectionMissing } from '../utils/appwriteErrors.mjs'
const router = express.Router()
@@ -165,30 +166,91 @@ router.post('/checkout',
/**
* GET /api/subscription/status
* Get user's subscription status
* (Also registered on app in index.mjs before router mount.)
*/
router.get('/status', asyncHandler(async (req, res) => {
export async function handleGetSubscriptionStatus(req, res) {
const userId = req.appwriteUser.id
const sub = await subscriptions.getByUser(userId, req.appwriteUser?.email)
const topKey = config.topSubscriptionPlan
const plan = sub.plan || topKey
const features =
PLANS[plan]?.features ||
PLANS[topKey]?.features ||
PLANS.business.features
if (isAdmin(req.appwriteUser.email)) {
return respond.success(res, {
status: 'active',
plan: 'business',
planDisplayName: 'Business (Admin)',
isFreeTier: false,
emailsUsedThisMonth: 0,
emailsLimit: 999999,
features: {
emailAccounts: 999,
emailsPerDay: 999999,
historicalSync: true,
customRules: true,
prioritySupport: true,
},
})
}
respond.success(res, {
status: sub.status || 'active',
plan,
planDisplayName: PLAN_DISPLAY_NAMES[plan] || PLAN_DISPLAY_NAMES[topKey] || 'Business',
isFreeTier: Boolean(sub.isFreeTier),
emailsUsedThisMonth: sub.emailsUsedThisMonth ?? 0,
emailsLimit: sub.emailsLimit ?? -1,
features,
currentPeriodEnd: sub.currentPeriodEnd,
cancelAtPeriodEnd: Boolean(sub.cancelAtPeriodEnd),
})
}))
try {
const sub = await subscriptions.getByUser(userId, req.appwriteUser?.email)
// No subscription document yet (synthetic free tier from DB layer) — safe defaults, not 404
if (!sub?.$id && sub?.plan === 'free') {
return respond.success(res, {
status: 'free',
plan: 'free',
planDisplayName: PLAN_DISPLAY_NAMES.free,
isFreeTier: true,
emailsUsedThisMonth: sub.emailsUsedThisMonth ?? 0,
emailsLimit: sub.emailsLimit ?? 500,
features: {
emailAccounts: 1,
emailsPerDay: 50,
historicalSync: false,
customRules: false,
prioritySupport: false,
},
})
}
const topKey = config.topSubscriptionPlan
const plan = sub.plan || topKey
const features =
PLANS[plan]?.features ||
PLANS[topKey]?.features ||
PLANS.business.features
return respond.success(res, {
status: sub.status || 'active',
plan,
planDisplayName: PLAN_DISPLAY_NAMES[plan] || PLAN_DISPLAY_NAMES[topKey] || 'Business',
isFreeTier: Boolean(sub.isFreeTier),
emailsUsedThisMonth: sub.emailsUsedThisMonth ?? 0,
emailsLimit: sub.emailsLimit ?? -1,
features,
currentPeriodEnd: sub.currentPeriodEnd,
cancelAtPeriodEnd: Boolean(sub.cancelAtPeriodEnd),
})
} catch (err) {
if (isAppwriteCollectionMissing(err)) {
return respond.success(res, {
status: 'free',
plan: 'free',
isFreeTier: true,
emailsUsedThisMonth: 0,
emailsLimit: 50,
features: {
emailAccounts: 1,
emailsPerDay: 50,
historicalSync: false,
customRules: false,
prioritySupport: false,
},
})
}
throw err
}
}
router.get('/status', asyncHandler(handleGetSubscriptionStatus))
/**
* POST /api/subscription/portal

308
server/services/ai-sorter.mjs
View File

@@ -7,6 +7,132 @@ import { Mistral } from '@mistralai/mistralai'
import { config } from '../config/index.mjs'
import { log } from '../middleware/logger.mjs'
function sleep(ms) {
return new Promise((r) => setTimeout(r, ms))
}
function is503Error(error) {
const status = error?.status ?? error?.statusCode ?? error?.response?.status
if (status === 503) return true
const msg = String(error?.message || '').toLowerCase()
return msg.includes('503') || msg.includes('service unavailable')
}
function isRetryableError(err) {
if (is503Error(err)) return true
const status = err?.status ?? err?.statusCode ?? err?.response?.status
if (status === 429) return true
const msg = (err?.message || '').toLowerCase()
return (
msg.includes('429') ||
msg.includes('rate limit') ||
msg.includes('too many requests')
)
}
/**
* Rule-based fallback when Mistral is unavailable or rate-limited.
* @param {{ from?: string, subject?: string, snippet?: string }} emailData
*/
export function ruleBasedCategory(emailData) {
const from = (emailData.from || '').toLowerCase()
const subject = (emailData.subject || '').toLowerCase()
const snippet = (emailData.snippet || '').toLowerCase()
// NEWSLETTERS — mass emails, unsubscribe links
if (
from.includes('noreply') ||
from.includes('no-reply') ||
from.includes('newsletter') ||
from.includes('marketing') ||
subject.includes('newsletter') ||
subject.includes('unsubscribe') ||
subject.includes('abbestellen')
) {
return 'newsletters'
}
// PROMOTIONS — sales, offers, discounts
if (
subject.includes('sale') ||
subject.includes('offer') ||
subject.includes('deal') ||
subject.includes('discount') ||
subject.includes('% off') ||
subject.includes('angebot') ||
subject.includes('rabatt') ||
from.includes('promo') ||
from.includes('deals') ||
from.includes('offers')
) {
return 'promotions'
}
// INVOICES — billing documents
if (
subject.includes('invoice') ||
subject.includes('rechnung') ||
subject.includes('payment') ||
subject.includes('zahlung') ||
subject.includes('bill ') ||
subject.includes('receipt') ||
subject.includes('quittung')
) {
return 'invoices'
}
// SECURITY — ONLY real security alerts (very specific)
if (
(subject.includes('security alert') ||
subject.includes('sign-in') ||
subject.includes('new login') ||
subject.includes('suspicious') ||
subject.includes('verify your') ||
subject.includes('2fa') ||
subject.includes('two-factor') ||
subject.includes('password reset') ||
(subject.includes('passwort') && subject.includes('zurücksetzen'))) &&
(from.includes('security') ||
from.includes('noreply') ||
from.includes('accounts') ||
from.includes('alerts'))
) {
return 'security'
}
// CALENDAR — meetings and events
if (
subject.includes('meeting') ||
subject.includes('invitation') ||
subject.includes('calendar') ||
subject.includes('appointment') ||
subject.includes('termin') ||
subject.includes('einladung') ||
subject.endsWith('.ics')
) {
return 'calendar'
}
// VIP — personal direct emails (not noreply, short subject)
if (
!from.includes('noreply') &&
!from.includes('no-reply') &&
!from.includes('newsletter') &&
!from.includes('info@') &&
subject.length < 60 &&
subject.length > 3
) {
return 'vip'
}
// DEFAULT — review (not security!)
return 'review'
}
/** Pace Mistral calls (IMAP sort uses these in email.mjs) */
export const AI_BATCH_CHUNK_SIZE = 5
export const AI_BATCH_CHUNK_DELAY_MS = 2000
/**
* Email categories with metadata
* Uses Gmail categories where available
@@ -67,7 +193,8 @@ const CATEGORIES = {
},
security: {
name: 'Security',
description: 'Security codes and notifications',
description:
'ONLY real account-security mail: login alerts (new sign-in, suspicious activity), password reset/change, 2FA/MFA codes, device verification. NOT marketing, shipping alerts, price drops, social notifications, or generic “notification” subjects.',
color: '#f44336',
gmailCategory: null,
action: 'inbox', // Keep in inbox (important!)
@@ -396,7 +523,12 @@ export class AISorterService {
*/
async categorize(email, preferences = {}) {
if (!this.enabled) {
return { category: 'review', confidence: 0, reason: 'AI not configured' }
return {
category: ruleBasedCategory(email),
confidence: 0,
reason: 'AI not configured',
assignedTo: null,
}
}
const { from, subject, snippet } = email
@@ -409,6 +541,13 @@ export class AISorterService {
AVAILABLE CATEGORIES:
${Object.entries(CATEGORIES).map(([key, cat]) => `- ${key}: ${cat.name} - ${cat.description}`).join('\n')}
CLASSIFICATION RULES (important):
- security: Use ONLY for genuine account safety: password reset/change, 2FA/MFA codes, new device login, suspicious sign-in warnings from the service itself. Do NOT use security for marketing, newsletters, order/shipping "alerts", price alerts, social network notifications, or anything that merely says "alert" or "notification".
- social: Social networks, likes, follows, mentions, friend requests, activity digests.
- newsletters: Recurring digests, blogs, Substack, product updates that are not personal.
- promotions: Sales, discounts, ads, deals.
- review: When unsure or mixed — prefer review over guessing security.
${preferenceContext}
EMAIL:
@@ -422,36 +561,78 @@ If the email is clearly FOR a specific worker (e.g. "für Max", "an Anna", subje
Respond ONLY with the JSON object.`
try {
const response = await this.client.chat.complete({
model: this.model,
messages: [{ role: 'user', content: prompt }],
temperature: 0.1,
maxTokens: 150,
responseFormat: { type: 'json_object' },
})
const content = response.choices[0]?.message?.content
const parseAndValidate = (content) => {
const result = JSON.parse(content)
// Validate category
if (!CATEGORIES[result.category]) {
result.category = 'review'
}
// Validate assignedTo against name labels (id or name)
if (result.assignedTo && preferences.nameLabels?.length) {
const match = preferences.nameLabels.find(
l => l.enabled && (l.id === result.assignedTo || l.name === result.assignedTo)
(l) => l.enabled && (l.id === result.assignedTo || l.name === result.assignedTo)
)
if (!match) result.assignedTo = null
else result.assignedTo = match.id || match.name
}
return result
} catch (error) {
log.error('AI categorization failed', { error: error.message })
return { category: 'review', confidence: 0, reason: 'Categorization error' }
}
let attempt = 0
let used503Backoff = false
while (true) {
try {
const response = await this.client.chat.complete({
model: this.model,
messages: [{ role: 'user', content: prompt }],
temperature: 0.1,
maxTokens: 150,
responseFormat: { type: 'json_object' },
})
const content = response.choices[0]?.message?.content
return parseAndValidate(content)
} catch (error) {
if (!isRetryableError(error)) {
log.error('AI categorization failed', { error: error.message })
return {
category: ruleBasedCategory(email),
confidence: 0,
reason: 'Categorization error',
assignedTo: null,
}
}
if (is503Error(error)) {
if (!used503Backoff) {
used503Backoff = true
log.warn('Mistral 503 (service unavailable), retry in 5s', { attempt: attempt + 1 })
await sleep(5000)
continue
}
log.warn('Mistral 503 after retry, using rule-based fallback')
return {
category: ruleBasedCategory(email),
confidence: 0,
reason: '503 — rule-based fallback',
assignedTo: null,
}
}
if (attempt >= 2) {
log.warn('Mistral rate limit after retries, using rule-based fallback')
return {
category: ruleBasedCategory(email),
confidence: 0,
reason: 'Rate limit — rule-based fallback',
assignedTo: null,
}
}
if (attempt === 0) {
log.warn('Mistral rate limit (429), retry in 2s', { attempt: attempt + 1 })
await sleep(2000)
} else {
log.warn('Mistral rate limit (429), retry in 5s', { attempt: attempt + 1 })
await sleep(5000)
}
attempt++
}
}
}
@@ -460,9 +641,14 @@ Respond ONLY with the JSON object.`
*/
async batchCategorize(emails, preferences = {}) {
if (!this.enabled || emails.length === 0) {
return emails.map(e => ({
return emails.map((e) => ({
email: e,
classification: { category: 'review', confidence: 0, reason: 'AI not available' },
classification: {
category: ruleBasedCategory(e),
confidence: 0,
reason: 'AI not available',
assignedTo: null,
},
}))
}
@@ -486,7 +672,9 @@ Respond ONLY with the JSON object.`
const prompt = `You are an email sorting assistant. Categorize the following ${emails.length} emails.
CATEGORIES:
${Object.entries(CATEGORIES).map(([key, cat]) => `${key}: ${cat.name}`).join(' | ')}
${Object.entries(CATEGORIES).map(([key, cat]) => `${key}: ${cat.name} — ${cat.description}`).join('\n')}
RULES: Use "security" ONLY for real account safety (password/2FA/login alerts). NOT for marketing alerts, shipping updates, or social notifications — use promotions, newsletters, social, or review instead.
${preferenceContext}
@@ -499,7 +687,7 @@ If an email is clearly FOR a specific worker, set assignedTo to that worker's id
Respond ONLY with the JSON array.`
try {
const runBatchRequest = async () => {
const response = await this.client.chat.complete({
model: this.model,
messages: [{ role: 'user', content: prompt }],
@@ -511,7 +699,6 @@ Respond ONLY with the JSON array.`
const content = response.choices[0]?.message?.content
let parsed
// Handle both array and object responses
try {
parsed = JSON.parse(content)
if (parsed.results) parsed = parsed.results
@@ -519,17 +706,16 @@ Respond ONLY with the JSON array.`
throw new Error('Not an array')
}
} catch {
// Fallback to individual processing
return this._fallbackBatch(emails, preferences)
}
return emails.map((email, i) => {
const result = parsed.find(r => r.index === i)
const result = parsed.find((r) => r.index === i)
const category = result?.category && CATEGORIES[result.category] ? result.category : 'review'
let assignedTo = result?.assignedTo || null
if (assignedTo && preferences.nameLabels?.length) {
const match = preferences.nameLabels.find(
l => l.enabled && (l.id === assignedTo || l.name === assignedTo)
(l) => l.enabled && (l.id === assignedTo || l.name === assignedTo)
)
assignedTo = match ? (match.id || match.name) : null
}
@@ -538,10 +724,68 @@ Respond ONLY with the JSON array.`
classification: { category, confidence: 0.8, reason: 'Batch', assignedTo },
}
})
} catch (error) {
log.error('Batch categorization failed', { error: error.message })
return this._fallbackBatch(emails, preferences)
}
let attempt = 0
let used503Backoff = false
while (true) {
try {
return await runBatchRequest()
} catch (error) {
if (!isRetryableError(error)) {
log.error('Batch categorization failed', { error: error.message })
return this._fallbackBatch(emails, preferences)
}
if (is503Error(error)) {
if (!used503Backoff) {
used503Backoff = true
log.warn('Mistral batch 503 (service unavailable), retry in 5s', { attempt: attempt + 1 })
await sleep(5000)
continue
}
log.warn('Mistral batch 503 after retry, rule-based per email')
return emails.map((email) => ({
email,
classification: {
category: ruleBasedCategory(email),
confidence: 0,
reason: '503 — rule-based fallback',
assignedTo: null,
},
}))
}
if (attempt >= 2) {
log.warn('Mistral batch rate limit after retries, rule-based per email')
return emails.map((email) => ({
email,
classification: {
category: ruleBasedCategory(email),
confidence: 0,
reason: 'Rate limit — rule-based fallback',
assignedTo: null,
},
}))
}
if (attempt === 0) {
log.warn('Mistral batch rate limit (429), retry in 2s', { attempt: attempt + 1 })
await sleep(2000)
} else {
log.warn('Mistral batch rate limit (429), retry in 5s', { attempt: attempt + 1 })
await sleep(5000)
}
attempt++
}
}
return emails.map((email) => ({
email,
classification: {
category: ruleBasedCategory(email),
confidence: 0,
reason: 'Rate limit — rule-based fallback',
assignedTo: null,
},
}))
}
/**

156
server/services/database.mjs
View File

@@ -6,6 +6,7 @@
import { Client, Databases, Query, ID } from 'node-appwrite'
import { config, isAdmin } from '../config/index.mjs'
import { NotFoundError } from '../middleware/errorHandler.mjs'
import { log } from '../middleware/logger.mjs'
// Initialize Appwrite client
const client = new Client()
@@ -15,6 +16,19 @@ const client = new Client()
const databases = new Databases(client)
const DB_ID = config.appwrite.databaseId
const DATABASE_ID = DB_ID
/**
* Appwrite: database/collection missing (schema not provisioned yet)
*/
function isCollectionNotFound(err) {
if (!err) return false
const msg = typeof err.message === 'string' ? err.message : ''
return (
err.type === 'collection_not_found' ||
msg.includes('Collection with the requested ID')
)
}
/**
* Collection names
@@ -44,7 +58,12 @@ export const db = {
* Create a document
*/
async create(collection, data, id = ID.unique()) {
return await databases.createDocument(DB_ID, collection, id, data)
try {
return await databases.createDocument(DB_ID, collection, id, data)
} catch (err) {
if (isCollectionNotFound(err)) return null
throw err
}
},
/**
@@ -54,6 +73,9 @@ export const db = {
try {
return await databases.getDocument(DB_ID, collection, id)
} catch (error) {
if (isCollectionNotFound(error)) {
return null
}
if (error.code === 404) {
throw new NotFoundError(collection)
}
@@ -65,22 +87,43 @@ export const db = {
* Update a document
*/
async update(collection, id, data) {
return await databases.updateDocument(DB_ID, collection, id, data)
try {
return await databases.updateDocument(DB_ID, collection, id, data)
} catch (err) {
if (isCollectionNotFound(err)) return null
throw err
}
},
/**
* Delete a document
*/
async delete(collection, id) {
return await databases.deleteDocument(DB_ID, collection, id)
try {
return await databases.deleteDocument(DB_ID, collection, id)
} catch (err) {
if (isCollectionNotFound(err)) return null
throw err
}
},
/**
* List documents with optional queries
*/
async list(collection, queries = []) {
const response = await databases.listDocuments(DB_ID, collection, queries)
return response.documents
async list(collectionId, queries = []) {
try {
const response = await databases.listDocuments(
DATABASE_ID,
collectionId,
queries
)
return response.documents
} catch (err) {
if (isCollectionNotFound(err)) {
return []
}
throw err
}
},
/**
@@ -98,7 +141,8 @@ export const db = {
try {
await databases.getDocument(DB_ID, collection, id)
return true
} catch {
} catch (err) {
if (isCollectionNotFound(err)) return false
return false
}
},
@@ -107,11 +151,16 @@ export const db = {
* Count documents
*/
async count(collection, queries = []) {
const response = await databases.listDocuments(DB_ID, collection, [
...queries,
Query.limit(1),
])
return response.total
try {
const response = await databases.listDocuments(DB_ID, collection, [
...queries,
Query.limit(1),
])
return response.total
} catch (err) {
if (isCollectionNotFound(err)) return 0
throw err
}
},
}
@@ -208,21 +257,28 @@ export const emailStats = {
const stats = await this.getByUser(userId)
if (stats) {
return db.update(Collections.EMAIL_STATS, stats.$id, {
const updated = await db.update(Collections.EMAIL_STATS, stats.$id, {
totalSorted: (stats.totalSorted || 0) + (counts.total || 0),
todaySorted: (stats.todaySorted || 0) + (counts.today || 0),
weekSorted: (stats.weekSorted || 0) + (counts.week || 0),
timeSavedMinutes: (stats.timeSavedMinutes || 0) + (counts.timeSaved || 0),
})
} else {
return this.create(userId, {
totalSorted: counts.total || 0,
todaySorted: counts.today || 0,
weekSorted: counts.week || 0,
timeSavedMinutes: counts.timeSaved || 0,
categoriesJson: '{}',
})
if (updated == null && process.env.NODE_ENV === 'development') {
log.warn('emailStats.increment: update skipped (missing collection or document)', { userId })
}
return updated
}
const created = await this.create(userId, {
totalSorted: counts.total || 0,
todaySorted: counts.today || 0,
weekSorted: counts.week || 0,
timeSavedMinutes: counts.timeSaved || 0,
categoriesJson: '{}',
})
if (created == null && process.env.NODE_ENV === 'development') {
log.warn('emailStats.increment: create skipped (missing collection)', { userId })
}
return created
},
async updateCategories(userId, categories) {
@@ -276,18 +332,26 @@ export const emailUsage = {
const existing = await this.getCurrentMonth(userId)
if (existing) {
return db.update(Collections.EMAIL_USAGE, existing.$id, {
const updated = await db.update(Collections.EMAIL_USAGE, existing.$id, {
emailsProcessed: (existing.emailsProcessed || 0) + count,
lastReset: new Date().toISOString(),
})
if (updated == null && process.env.NODE_ENV === 'development') {
log.warn('emailUsage.increment: update skipped (missing collection or document)', { userId })
}
return updated
}
return db.create(Collections.EMAIL_USAGE, {
const created = await db.create(Collections.EMAIL_USAGE, {
userId,
month,
emailsProcessed: count,
lastReset: new Date().toISOString(),
})
if (created == null && process.env.NODE_ENV === 'development') {
log.warn('emailUsage.increment: create skipped (missing collection)', { userId })
}
return created
},
async getUsage(userId) {
@@ -578,6 +642,26 @@ export const onboardingState = {
last_updated: new Date().toISOString(),
})
},
/**
* Reset onboarding to initial state (does not delete the document).
*/
async resetToInitial(userId) {
const existing = await db.findOne(Collections.ONBOARDING_STATE, [
Query.equal('userId', userId),
])
const data = {
onboarding_step: 'not_started',
completed_steps_json: JSON.stringify([]),
skipped_at: null,
first_value_seen_at: null,
last_updated: new Date().toISOString(),
}
if (existing) {
return db.update(Collections.ONBOARDING_STATE, existing.$id, data)
}
return db.create(Collections.ONBOARDING_STATE, { userId, ...data })
},
}
/**
@@ -609,12 +693,17 @@ export const referrals = {
attempts++
}
return db.create(Collections.REFERRALS, {
const created = await db.create(Collections.REFERRALS, {
userId,
referralCode: uniqueCode,
referralCount: 0,
createdAt: new Date().toISOString(),
})
// Collection missing → return null safely
if (!created) return null
return created
},
async getByCode(code) {
@@ -746,6 +835,25 @@ export const emailDigests = {
},
}
/**
* Delete all documents in a collection where attribute `userId` matches (paginated batches of 100).
*/
export async function deleteAllDocumentsForUser(collection, userId) {
let deleted = 0
for (;;) {
const batch = await db.list(collection, [
Query.equal('userId', userId),
Query.limit(100),
])
if (!batch.length) break
for (const doc of batch) {
await db.delete(collection, doc.$id)
deleted++
}
}
return deleted
}
export { Query }
export default {

711
server/services/imap.mjs
View File

@@ -9,6 +9,33 @@ import { log } from '../middleware/logger.mjs'
const INBOX = 'INBOX'
const FOLDER_PREFIX = 'MailFlow'
/** Timeouts (ms) — avoid hanging on bad host/firewall */
const IMAP_CONNECT_TIMEOUT_MS = 10_000
const IMAP_OPERATION_TIMEOUT_MS = 10_000
/** Per-command cap so dead connections fail fast (sort try/catch + race can complete). */
const IMAP_OP_TIMEOUT_MS = 5000
async function withOpTimeout(promise, label = 'IMAP op') {
return Promise.race([
promise,
new Promise((_, reject) =>
setTimeout(() => reject(new Error(`${label} timed out`)), IMAP_OP_TIMEOUT_MS)
),
])
}
function rejectAfter(ms, message) {
return new Promise((_, reject) => {
setTimeout(() => reject(new Error(message)), ms)
})
}
async function withTimeout(promise, ms, label) {
const msg = `${label} timed out after ${ms / 1000} seconds`
return Promise.race([promise, rejectAfter(ms, msg)])
}
/** Map category key to IMAP folder name */
export function getFolderNameForCategory(category) {
const map = {
@@ -26,6 +53,212 @@ export function getFolderNameForCategory(category) {
return map[category] || 'Review'
}
/**
* Folder last segments that must NEVER be used as sort targets (Trash, Sent, Drafts, Deleted only).
* Junk, Spam, Archive, Favorites are valid targets.
*/
const FORBIDDEN_LAST_SEGMENTS = new Set(['trash', 'sent', 'drafts', 'deleted'])
/** Full paths that are never valid move targets (case-insensitive). */
const FORBIDDEN_FULL_PATHS = new Set([
'trash',
'sent',
'drafts',
'deleted',
'inbox.trash',
'inbox.sent',
'inbox.drafts',
'inbox.deleted',
])
/**
* Keywords per AI category for matching existing mailbox paths (case-insensitive).
* First match wins (keyword order, then folder list order).
*/
export const CATEGORY_FOLDER_KEYWORDS = {
vip: ['favorites', 'vip', 'important', 'priority', 'wichtig'],
customers: ['clients', 'customers', 'kunden', 'client'],
invoices: ['invoices', 'invoice', 'rechnungen', 'rechnung', 'billing'],
newsletters: ['junk', 'newsletters', 'newsletter', 'subscriptions'],
promotions: ['junk', 'promotions', 'promotion', 'marketing', 'spam'],
social: ['social', 'notifications', 'team'],
security: ['security', 'alerts', 'sicherheit'],
calendar: ['calendar', 'meetings'],
review: ['archive', 'review', 'later'],
}
function lastMailboxSegment(path) {
if (!path || typeof path !== 'string') return ''
const parts = path.split(/[/\\]/).filter(Boolean)
const leaf = parts.length ? parts[parts.length - 1] : path
const dotted = leaf.split('.')
return dotted[dotted.length - 1].toLowerCase()
}
/**
* True if this path must not be used as a sort move destination.
*/
export function isForbiddenMoveTarget(folderPath) {
if (!folderPath || typeof folderPath !== 'string') return true
const norm = folderPath.trim()
if (!norm) return true
if (norm.toUpperCase() === 'INBOX') return true
const fullLower = norm.toLowerCase().replace(/\\/g, '/')
if (FORBIDDEN_FULL_PATHS.has(fullLower)) return true
const last = lastMailboxSegment(norm)
return FORBIDDEN_LAST_SEGMENTS.has(last)
}
/**
* Pick the best existing folder for a category, or null (keep in INBOX).
* Prefers exact last-segment match (e.g. path "Junk" or "*.Junk") over substring (avoids wrong "junk" hits).
* @param {string} category
* @param {string[]} existingFolders - paths from LIST
* @returns {string | null}
*/
export function findBestFolder(category, existingFolders) {
if (!existingFolders?.length) return null
const keywords = CATEGORY_FOLDER_KEYWORDS[category]
if (!keywords?.length) return null
for (const keyword of keywords) {
const kw = keyword.toLowerCase()
for (const folderPath of existingFolders) {
if (!folderPath || isForbiddenMoveTarget(folderPath)) continue
if (lastMailboxSegment(folderPath) === kw) return folderPath
}
}
for (const keyword of keywords) {
const kw = keyword.toLowerCase()
for (const folderPath of existingFolders) {
if (!folderPath || isForbiddenMoveTarget(folderPath)) continue
const pathLower = folderPath.toLowerCase()
if (pathLower.includes(kw)) return folderPath
}
}
return null
}
/** System mailboxes we never pull from during "re-sort" recovery (keep Sent/Drafts/Trash untouched). */
const RESORT_SKIP_LAST_SEGMENTS = new Set([
'sent',
'drafts',
'trash',
'deleted',
'templates',
'outbox',
'inbox',
])
/**
* Folders whose messages we move back to INBOX before a full re-sort (sort destinations + MailFlow/EmailSorter trees).
* @param {string} folderPath
* @returns {boolean}
*/
export function isReSortRecoveryFolder(folderPath) {
if (!folderPath || typeof folderPath !== 'string') return false
const norm = folderPath.trim()
if (!norm || norm.toUpperCase() === 'INBOX') return false
const last = lastMailboxSegment(norm)
if (RESORT_SKIP_LAST_SEGMENTS.has(last)) return false
const low = norm.toLowerCase().replace(/\\/g, '/')
if (low.includes('mailflow') || low.includes('emailsorter')) return true
const sortLeaves = new Set([
'junk',
'spam',
'archive',
'newsletters',
'newsletter',
'promotions',
'promotion',
'social',
'review',
'security',
'calendar',
'invoices',
'invoice',
'clients',
'customers',
'client',
'vip',
'favorites',
'important',
'subscriptions',
'marketing',
'team',
'meetings',
'later',
'billing',
'rechnungen',
'rechnung',
])
return sortLeaves.has(last)
}
/**
* Match a person/team folder by local name appearing in From or Subject.
* @param {{ from?: string, subject?: string }} emailData
* @param {string[]} existingFolders
* @returns {string | null}
*/
export function findPersonFolder(emailData, existingFolders) {
const from = (emailData.from || '').toLowerCase()
const subject = (emailData.subject || '').toLowerCase()
for (const folder of existingFolders) {
const folderName = folder.split('.').pop() || folder
if (folderName.length < 3) continue
const skip = [
'inbox',
'sent',
'trash',
'junk',
'spam',
'drafts',
'archive',
'deleted',
'favorites',
'emailsorter',
'vip',
'clients',
'invoices',
'newsletters',
'promotions',
'security',
'calendar',
'review',
]
if (skip.includes(folderName.toLowerCase())) continue
const name = folderName.toLowerCase()
if (from.includes(name) || subject.includes(name)) {
return folder
}
}
return null
}
/** IMAP keywords ($MailFlow-*) — reliable on hosts that block custom folders */
export function getMailFlowKeywordForCategory(category) {
const map = {
vip: '$MailFlow-VIP',
customers: '$MailFlow-Clients',
invoices: '$MailFlow-Invoices',
newsletters: '$MailFlow-Newsletters',
promotions: '$MailFlow-Promotions',
social: '$MailFlow-Social',
security: '$MailFlow-Security',
calendar: '$MailFlow-Calendar',
review: '$MailFlow-Review',
archive: '$MailFlow-Archive',
}
return map[category] || '$MailFlow-Review'
}
/**
* IMAP Service – same conceptual interface as GmailService/OutlookService
*/
@@ -40,18 +273,47 @@ export class ImapService {
*/
constructor(opts) {
const { host, port = 993, secure = true, user, password } = opts
this.client = new ImapFlow({
host: host || 'imap.porkbun.com',
port: port || 993,
this.host = host || 'imap.porkbun.com'
this.port = port || 993
this._imapFlowOptions = {
host: this.host,
port: this.port,
secure: secure !== false,
auth: { user, pass: password },
logger: false,
})
}
this.client = new ImapFlow({ ...this._imapFlowOptions })
this.lock = null
}
async ensureConnected() {
try {
if (!this.client.usable) throw new Error('not usable')
await withOpTimeout(this.client.noop(), 'noop')
} catch {
try {
if (this.lock) await this.lock.release().catch(() => {})
} catch {
/* ignore */
}
this.lock = null
try {
await this.client.logout()
} catch {
try {
this.client.close()
} catch {
/* ignore */
}
}
this.client = new ImapFlow({ ...this._imapFlowOptions })
await this.connect()
}
}
async connect() {
await this.client.connect()
console.log('[IMAP] Connecting to:', this.host, this.port)
await withTimeout(this.client.connect(), IMAP_CONNECT_TIMEOUT_MS, 'IMAP connect')
}
async close() {
@@ -69,6 +331,10 @@ export class ImapService {
* @param {string|null} _pageToken - reserved for future pagination
*/
async listEmails(maxResults = 50, _pageToken = null) {
return withTimeout(this._listEmailsImpl(maxResults, _pageToken), IMAP_OPERATION_TIMEOUT_MS, 'IMAP listEmails')
}
async _listEmailsImpl(maxResults = 50, _pageToken = null) {
let lock = null
try {
lock = await this.client.getMailboxLock(INBOX)
@@ -123,6 +389,10 @@ export class ImapService {
*/
async batchGetEmails(messageIds) {
if (!messageIds.length) return []
return withTimeout(this._batchGetEmailsImpl(messageIds), IMAP_OPERATION_TIMEOUT_MS, 'IMAP batchGetEmails')
}
async _batchGetEmailsImpl(messageIds) {
let lock = null
try {
lock = await this.client.getMailboxLock(INBOX)
@@ -142,13 +412,26 @@ export class ImapService {
}
}
/**
* List all mailbox paths on the server (for sort: map categories to existing folders only).
* @returns {Promise<string[]>}
*/
async listAllFolders() {
await this.ensureConnected()
const allFolders = await withOpTimeout(this.client.list(), 'list')
return (allFolders || [])
.map((f) => f.path || f.name)
.filter((p) => typeof p === 'string' && p.length > 0)
}
/**
* Ensure folder exists (create if not). Use subfolder under MailFlow to avoid clutter.
* NOT USED IN SORT — automatic sort must not create folders; use existing paths only.
*/
async ensureFolder(folderName) {
const path = `${FOLDER_PREFIX}/${folderName}`
try {
await this.client.mailboxCreate(path)
await withOpTimeout(this.client.mailboxCreate(path), 'mailboxCreate')
log.info(`IMAP folder created: ${path}`)
} catch (err) {
if (err.code !== 'ALREADYEXISTS' && !/already exists/i.test(err.message)) {
@@ -159,16 +442,274 @@ export class ImapService {
}
/**
* NOT USED IN SORT — sort uses `copyToFolder` (no folder creation; originals stay in INBOX).
* Move message (by UID) from INBOX to folder name (under MailFlow/)
*/
async moveToFolder(messageId, folderName) {
await this.ensureConnected()
const path = `${FOLDER_PREFIX}/${folderName}`
await this.ensureFolder(folderName)
const runMove = async () => {
let lock = null
try {
lock = await withOpTimeout(this.client.getMailboxLock(INBOX), 'getMailboxLock')
this.lock = lock
await withOpTimeout(
this.client.messageMove(String(messageId), path, { uid: true }),
'messageMove'
)
} finally {
if (lock) {
lock.release()
this.lock = null
}
}
}
try {
await this.ensureFolder(folderName)
await runMove()
} catch (err) {
log.warn('IMAP moveToFolder first attempt failed', { error: err.message })
await this.ensureConnected()
await this.ensureFolder(folderName)
await runMove()
}
}
/**
* Add MailFlow category keyword (and optional team tag) on INBOX message — no folder required.
* Never throws; returns false if tagging failed (sort can still count the email as categorized).
* @param {string | null} [assignedTo]
* @returns {Promise<boolean>}
*/
async addMailFlowCategoryKeyword(messageId, category, assignedTo = null) {
const flags = [getMailFlowKeywordForCategory(category)]
if (assignedTo) {
const safe = String(assignedTo).replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 50)
flags.push(`$MailFlow-Team-${safe}`)
}
try {
await this.ensureConnected()
let lock = null
try {
lock = await withOpTimeout(this.client.getMailboxLock(INBOX), 'getMailboxLock')
this.lock = lock
await withOpTimeout(
this.client.messageFlagsAdd(String(messageId), flags, { uid: true }),
'messageFlagsAdd'
)
} finally {
if (lock) {
lock.release()
this.lock = null
}
}
return true
} catch (err) {
const msg = err?.message || String(err)
log.warn('IMAP addMailFlowCategoryKeyword failed', { messageId, error: msg })
return false
}
}
/**
* Move UID from INBOX to an existing mailbox path only. Does not create folders.
* IMAP sort (POST /sort) uses `copyToFolder` instead — originals stay in INBOX.
* Returns true on success; false on validation failure or IMAP error (never throws).
* @param {string} messageId - UID string
* @param {string} destPath - exact path from LIST
* @param {Set<string>} existingFolderSet - paths that exist on server
*/
async moveMessageToExistingPath(messageId, destPath, existingFolderSet) {
const uid = typeof messageId === 'string' ? Number(messageId) : Number(messageId)
if (messageId == null || Number.isNaN(uid) || uid < 1) {
log.warn('IMAP moveMessageToExistingPath: invalid UID', { messageId })
return false
}
if (!destPath || !existingFolderSet || !(existingFolderSet instanceof Set)) {
log.warn('IMAP moveMessageToExistingPath: bad dest or set', { destPath })
return false
}
if (!existingFolderSet.has(destPath)) {
log.warn('IMAP moveMessageToExistingPath: path not in existing set', { destPath })
return false
}
if (isForbiddenMoveTarget(destPath)) {
log.warn('IMAP moveMessageToExistingPath: forbidden destination', { destPath })
return false
}
try {
await this.ensureConnected()
let lock = null
try {
lock = await withOpTimeout(this.client.getMailboxLock(INBOX), 'getMailboxLock')
this.lock = lock
await withOpTimeout(
this.client.messageMove(String(uid), destPath, { uid: true }),
'messageMove'
)
const uidRange = `${uid}:${uid}`
const stillInInbox = await withOpTimeout(
this.client.search({ uid: uidRange }, { uid: true }),
'search'
)
if (stillInInbox?.length) {
log.warn('IMAP moveMessageToExistingPath: UID still in INBOX after MOVE (path may be wrong)', {
messageId,
destPath,
uid,
})
return false
}
return true
} finally {
if (lock) {
lock.release()
this.lock = null
}
}
} catch (err) {
log.warn('IMAP moveMessageToExistingPath failed', { messageId, destPath, error: err?.message })
return false
}
}
/**
* Copy UID from INBOX to an existing mailbox path (original stays in INBOX).
* Returns true on success; false on validation failure or IMAP error (never throws).
* @param {string} messageId - UID string
* @param {string} destPath - exact path from LIST
* @param {Set<string>} existingFolderSet - paths that exist on server
*/
async copyToFolder(messageId, destPath, existingFolderSet) {
const uid = typeof messageId === 'string' ? Number(messageId) : Number(messageId)
if (messageId == null || Number.isNaN(uid) || uid < 1) {
log.warn('IMAP copyToFolder: invalid UID', { messageId })
return false
}
if (!destPath || !existingFolderSet || !(existingFolderSet instanceof Set)) {
log.warn('IMAP copyToFolder: bad dest or set', { destPath })
return false
}
if (!existingFolderSet.has(destPath)) {
log.warn('IMAP copyToFolder: path not in existing set', { destPath })
return false
}
if (isForbiddenMoveTarget(destPath)) {
log.warn('IMAP copyToFolder: forbidden destination', { destPath })
return false
}
try {
await this.ensureConnected()
let lock = null
try {
lock = await withOpTimeout(this.client.getMailboxLock(INBOX), 'getMailboxLock')
this.lock = lock
await withOpTimeout(
this.client.messageCopy(String(uid), destPath, { uid: true }),
'messageCopy'
)
return true
} finally {
if (lock) {
lock.release()
this.lock = null
}
}
} catch (err) {
log.warn('IMAP copyToFolder failed', { messageId, destPath, error: err?.message })
return false
}
}
/**
* Remove $MailFlow-sorted keyword from all messages in INBOX (UID SEARCH + STORE).
* @returns {Promise<number>} count of UIDs processed (same as messages touched)
*/
async removeAllSortedFlags() {
await this.connect()
let lock = null
try {
lock = await this.client.getMailboxLock(INBOX)
lock = await withOpTimeout(this.client.getMailboxLock(INBOX), 'getMailboxLock')
this.lock = lock
await this.client.messageMove(String(messageId), path, { uid: true })
const allUids = await withOpTimeout(
this.client.search({ all: true }, { uid: true }),
'search'
)
const list = (allUids || [])
.map((u) => (typeof u === 'bigint' ? Number(u) : u))
.filter((n) => n != null && !Number.isNaN(Number(n)))
if (list.length > 0) {
await withOpTimeout(
this.client.messageFlagsRemove(list, ['$MailFlow-sorted'], { uid: true }),
'messageFlagsRemove'
)
}
log.info(`Removed $MailFlow-sorted from ${list.length} emails`)
return list.length
} catch (err) {
log.warn('IMAP removeAllSortedFlags failed', { error: err?.message })
return 0
} finally {
if (lock) {
lock.release()
this.lock = null
}
try {
await this.close()
} catch {
/* ignore */
}
}
}
/**
* NOT USED IN SORT — sort never moves to Trash/Archive for cleanup.
* Move to Trash or Archive (first that works) — standard mailboxes on most hosts.
*/
async moveToArchiveOrTrash(messageId) {
const candidates = ['Trash', 'Archive', 'Deleted', 'INBOX.Trash']
let lastErr = null
for (const dest of candidates) {
try {
await this.ensureConnected()
let lock = null
try {
lock = await withOpTimeout(this.client.getMailboxLock(INBOX), 'getMailboxLock')
this.lock = lock
await withOpTimeout(
this.client.messageMove(String(messageId), dest, { uid: true }),
'messageMove'
)
return
} finally {
if (lock) {
lock.release()
this.lock = null
}
}
} catch (err) {
lastErr = err
log.warn(`IMAP moveToArchiveOrTrash: ${dest} failed`, { error: err.message })
}
}
throw lastErr || new Error('IMAP move to Trash/Archive failed')
}
/**
* Mark message as read (\\Seen)
*/
async markAsRead(messageId) {
await this.ensureConnected()
let lock = null
try {
lock = await withOpTimeout(this.client.getMailboxLock(INBOX), 'getMailboxLock')
this.lock = lock
await withOpTimeout(
this.client.messageFlagsAdd(String(messageId), ['\\Seen'], { uid: true }),
'messageFlagsAdd'
)
} finally {
if (lock) {
lock.release()
@@ -178,14 +719,48 @@ export class ImapService {
}
/**
* Mark message as read (\\Seen)
* Remove every $MailFlow-* keyword from all messages in INBOX (incl. $MailFlow-sorted and $MailFlow-Team-*).
* Does not close the connection.
* @returns {Promise<number>} number of messages that had at least one MailFlow keyword removed
*/
async markAsRead(messageId) {
async stripAllMailFlowKeywordsInInbox() {
await this.ensureConnected()
let lock = null
let touched = 0
try {
lock = await this.client.getMailboxLock(INBOX)
lock = await withOpTimeout(this.client.getMailboxLock(INBOX), 'getMailboxLock')
this.lock = lock
await this.client.messageFlagsAdd(String(messageId), ['\\Seen'], { uid: true })
const allUids = await withOpTimeout(
this.client.search({ all: true }, { uid: true }),
'search'
)
const uidList = (allUids || [])
.map((u) => (typeof u === 'bigint' ? Number(u) : u))
.filter((n) => n != null && !Number.isNaN(Number(n)))
if (!uidList.length) return 0
const messages = await withOpTimeout(
this.client.fetchAll(uidList, { flags: true, uid: true }),
'fetchAll'
)
for (const msg of messages || []) {
const flagSet = msg.flags
if (!flagSet || !flagSet.size) continue
const toRemove = [...flagSet].filter(
(f) => typeof f === 'string' && f.startsWith('$MailFlow')
)
if (!toRemove.length) continue
await withOpTimeout(
this.client.messageFlagsRemove(String(msg.uid), toRemove, { uid: true }),
'messageFlagsRemove'
)
touched++
}
log.info(`Stripped MailFlow keywords from ${touched} INBOX message(s)`)
return touched
} catch (err) {
log.warn('IMAP stripAllMailFlowKeywordsInInbox failed', { error: err?.message })
return touched
} finally {
if (lock) {
lock.release()
@@ -193,4 +768,114 @@ export class ImapService {
}
}
}
/**
* Move messages from sort-related folders back to INBOX, then strip $MailFlow-* keywords in INBOX.
* Safer than recoverAllToInbox (does not touch Sent/Drafts/Trash/Deleted).
* Caller should connect first; does not call logout/close.
* @returns {Promise<{ recovered: number, folders: Array<{ folder: string, count: number }>, mailFlowKeywordsStripped: number }>}
*/
async reSortRecoverAndStripKeywords() {
await this.ensureConnected()
let recovered = 0
const folders = []
try {
const listed = await withOpTimeout(this.client.list(), 'list')
for (const folder of listed || []) {
const name = folder.path || folder.name
if (!name || !isReSortRecoveryFolder(name)) continue
if (folder.flags?.has?.('\\Noselect') || folder.flags?.has?.('\\NonExistent')) continue
try {
const lock = await withOpTimeout(this.client.getMailboxLock(name), 'getMailboxLock')
this.lock = lock
try {
const uids = await withOpTimeout(
this.client.search({ all: true }, { uid: true }),
'search'
)
if (!uids?.length) continue
const uidList = uids
.map((u) => (typeof u === 'bigint' ? Number(u) : u))
.filter((n) => n != null && !Number.isNaN(Number(n)))
if (!uidList.length) continue
folders.push({ folder: name, count: uidList.length })
await withOpTimeout(
this.client.messageMove(uidList, INBOX, { uid: true }),
'messageMove'
)
recovered += uidList.length
log.info(`Re-sort recovery: ${uidList.length} message(s) from "${name}" → INBOX`)
} finally {
lock.release()
this.lock = null
}
} catch (err) {
log.warn(`Re-sort: could not empty folder "${name}"`, { error: err?.message })
}
}
const mailFlowKeywordsStripped = await this.stripAllMailFlowKeywordsInInbox()
return { recovered, folders, mailFlowKeywordsStripped }
} catch (err) {
log.warn('IMAP reSortRecoverAndStripKeywords failed', { error: err?.message })
return { recovered, folders, mailFlowKeywordsStripped: 0 }
}
}
/**
* Move all messages from every mailbox except INBOX into INBOX (nuclear recovery).
* WARNING: Also affects Sent, Drafts, Trash, Junk, etc. — only use to recover mail misplaced by buggy moves.
*/
async recoverAllToInbox() {
await this.connect()
const checkedFolders = []
let recovered = 0
try {
const allFolders = await withOpTimeout(this.client.list(), 'list')
for (const folder of allFolders) {
const name = folder.path || folder.name
if (!name || name.toUpperCase() === 'INBOX') continue
try {
const lock = await withOpTimeout(this.client.getMailboxLock(name), 'getMailboxLock')
this.lock = lock
try {
const uids = await withOpTimeout(
this.client.search({ all: true }, { uid: true }),
'search'
)
if (!uids || !uids.length) continue
const uidList = uids
.map((u) => (typeof u === 'bigint' ? Number(u) : u))
.filter((n) => n != null && !Number.isNaN(Number(n)))
if (!uidList.length) continue
checkedFolders.push({ folder: name, count: uidList.length })
await withOpTimeout(
this.client.messageMove(uidList, INBOX, { uid: true }),
'messageMove'
)
recovered += uidList.length
log.info(`Recovered ${uidList.length} emails from "${name}" → INBOX`)
} finally {
lock.release()
this.lock = null
}
} catch (err) {
log.warn(`Could not process folder "${name}"`, { error: err.message })
}
}
} finally {
try {
await this.close()
} catch {
/* ignore */
}
}
return { recovered, folders: checkedFolders }
}
}

12
server/utils/appwriteErrors.mjs Normal file
View File

@@ -0,0 +1,12 @@
/**
* Detect Appwrite errors when a collection/database resource is missing.
*/
export function isAppwriteCollectionMissing(err) {
if (!err) return false
const msg = typeof err.message === 'string' ? err.message : ''
return (
err.code === 404 ||
err.type === 'collection_not_found' ||
msg.includes('Collection with the requested ID')
)
}