From a982a5a57b7aab2e8326e79c2f0eab7ebafeff5f Mon Sep 17 00:00:00 2001 From: shuaiplus <2327005759@qq.com> Date: Thu, 9 Apr 2026 23:05:00 +0800 Subject: [PATCH] feat: enhance database indexing and optimize sync response handling --- migrations/0001_init.sql | 2 + src/config/limits.ts | 3 + src/handlers/accounts.ts | 3 +- src/handlers/attachments.ts | 17 +++- src/handlers/ciphers.ts | 8 +- src/handlers/identity.ts | 20 ++-- src/handlers/sends-private.ts | 3 +- src/handlers/sync.ts | 159 ++++++++++--------------------- src/services/storage-schema.ts | 2 + webapp/src/lib/api/send.ts | 7 +- webapp/src/lib/api/vault-sync.ts | 31 ++++++ webapp/src/lib/api/vault.ts | 14 +-- 12 files changed, 129 insertions(+), 140 deletions(-) create mode 100644 webapp/src/lib/api/vault-sync.ts diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql index 8053368..3d1da10 100644 --- a/migrations/0001_init.sql +++ b/migrations/0001_init.sql @@ -59,6 +59,7 @@ CREATE TABLE IF NOT EXISTS ciphers ( CREATE INDEX IF NOT EXISTS idx_ciphers_user_updated ON ciphers(user_id, updated_at); CREATE INDEX IF NOT EXISTS idx_ciphers_user_archived ON ciphers(user_id, archived_at); CREATE INDEX IF NOT EXISTS idx_ciphers_user_deleted ON ciphers(user_id, deleted_at); +CREATE INDEX IF NOT EXISTS idx_ciphers_user_deleted_updated ON ciphers(user_id, deleted_at, updated_at); CREATE TABLE IF NOT EXISTS folders ( id TEXT PRIMARY KEY, @@ -106,6 +107,7 @@ CREATE TABLE IF NOT EXISTS sends ( ); CREATE INDEX IF NOT EXISTS idx_sends_user_updated ON sends(user_id, updated_at); CREATE INDEX IF NOT EXISTS idx_sends_user_deletion ON sends(user_id, deletion_date); +CREATE INDEX IF NOT EXISTS idx_sends_user_updated_id ON sends(user_id, updated_at, id); CREATE TABLE IF NOT EXISTS refresh_tokens ( token TEXT PRIMARY KEY, diff --git a/src/config/limits.ts b/src/config/limits.ts index b85d275..8eb4a4c 100644 --- a/src/config/limits.ts +++ b/src/config/limits.ts @@ -130,6 +130,9 @@ // Max total items (folders + ciphers) allowed in a single import. // 单次导入允许的最大条目数（文件夹 + 密码项合计）。 importItemLimit: 5000, + // Small fixed concurrency for blob/attachment batch cleanup work. + // 附件 / blob 批量清理时的保守并发数。 + attachmentDeleteConcurrency: 4, }, request: { // Hard body size limit for JSON API endpoints (bytes). File upload paths are exempt. diff --git a/src/handlers/accounts.ts b/src/handlers/accounts.ts index b7d5603..9b4204b 100644 --- a/src/handlers/accounts.ts +++ b/src/handlers/accounts.ts @@ -87,6 +87,7 @@ async function verifyUserSecret( function toProfile(user: User, env: Env): ProfileResponse { void env; + const accountKeys = buildAccountKeys(user); return { id: user.id, name: user.name, @@ -100,7 +101,7 @@ function toProfile(user: User, env: Env): ProfileResponse { twoFactorEnabled: !!user.totpSecret, key: user.key, privateKey: user.privateKey, - accountKeys: buildAccountKeys(user), + accountKeys, securityStamp: user.securityStamp || user.id, organizations: [], providers: [], diff --git a/src/handlers/attachments.ts b/src/handlers/attachments.ts index 403b25f..91a81d2 100644 --- a/src/handlers/attachments.ts +++ b/src/handlers/attachments.ts @@ -38,6 +38,18 @@ function formatSize(bytes: number): string { return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; } +async function runWithConcurrency( + items: T[], + concurrency: number, + worker: (item: T) => Promise +): Promise { + if (items.length === 0) return; + const limit = Math.max(1, concurrency); + for (let index = 0; index < items.length; index += limit) { + await Promise.all(items.slice(index, index + limit).map(worker)); + } +} + async function processAttachmentUpload( request: Request, env: Env, @@ -381,10 +393,9 @@ export async function deleteAllAttachmentsForCipher( ): Promise { const storage = new StorageService(env.DB); const attachments = await storage.getAttachmentsByCipher(cipherId); - - for (const attachment of attachments) { + await runWithConcurrency(attachments, LIMITS.performance.attachmentDeleteConcurrency, async (attachment) => { const path = getAttachmentObjectKey(cipherId, attachment.id); await deleteBlobObject(env, path); await storage.deleteAttachment(attachment.id); - } + }); } diff --git a/src/handlers/ciphers.ts b/src/handlers/ciphers.ts index fec3e46..ff3c63b 100644 --- a/src/handlers/ciphers.ts +++ b/src/handlers/ciphers.ts @@ -178,10 +178,12 @@ export async function handleGetCiphers(request: Request, env: Env, userId: strin : ciphers.filter(c => !c.deletedAt); } - const attachmentsByCipher = await storage.getAttachmentsByUserId(userId); + const attachmentsByCipher = await storage.getAttachmentsByCipherIds( + filteredCiphers.map((cipher) => cipher.id) + ); - // Get attachments for all ciphers - const cipherResponses = []; + // Build responses only for the current page to keep pagination cheap. + const cipherResponses: CipherResponse[] = []; for (const cipher of filteredCiphers) { const attachments = attachmentsByCipher.get(cipher.id) || []; cipherResponses.push(cipherToResponse(cipher, attachments)); diff --git a/src/handlers/identity.ts b/src/handlers/identity.ts index 41ecd45..271535b 100644 --- a/src/handlers/identity.ts +++ b/src/handlers/identity.ts @@ -327,6 +327,8 @@ export async function handleToken(request: Request, env: Env): Promise const accessToken = await auth.generateAccessToken(user, deviceSession); const refreshToken = await auth.generateRefreshToken(user.id, deviceSession); + const accountKeys = buildAccountKeys(user); + const userDecryptionOptions = buildUserDecryptionOptions(user); const response: TokenResponse = { access_token: accessToken, @@ -336,8 +338,8 @@ export async function handleToken(request: Request, env: Env): Promise ...(trustedTwoFactorTokenToReturn ? { TwoFactorToken: trustedTwoFactorTokenToReturn } : {}), Key: user.key, PrivateKey: user.privateKey, - AccountKeys: buildAccountKeys(user), - accountKeys: buildAccountKeys(user), + AccountKeys: accountKeys, + accountKeys: accountKeys, Kdf: user.kdfType, KdfIterations: user.kdfIterations, KdfMemory: user.kdfMemory, @@ -350,8 +352,8 @@ export async function handleToken(request: Request, env: Env): Promise ApiUseKeyConnector: false, scope: 'api offline_access', unofficialServer: true, - UserDecryptionOptions: buildUserDecryptionOptions(user), - userDecryptionOptions: buildUserDecryptionOptions(user), + UserDecryptionOptions: userDecryptionOptions, + userDecryptionOptions: userDecryptionOptions, }; const baseResponse = jsonResponse(response); @@ -449,6 +451,8 @@ export async function handleToken(request: Request, env: Env): Promise const { accessToken, user, device } = result; const newRefreshToken = await auth.generateRefreshToken(user.id, device); + const accountKeys = buildAccountKeys(user); + const userDecryptionOptions = buildUserDecryptionOptions(user); const response: TokenResponse = { access_token: accessToken, @@ -457,8 +461,8 @@ export async function handleToken(request: Request, env: Env): Promise ...(shouldUseWebSession(request) ? { web_session: true } : { refresh_token: newRefreshToken }), Key: user.key, PrivateKey: user.privateKey, - AccountKeys: buildAccountKeys(user), - accountKeys: buildAccountKeys(user), + AccountKeys: accountKeys, + accountKeys: accountKeys, Kdf: user.kdfType, KdfIterations: user.kdfIterations, KdfMemory: user.kdfMemory, @@ -471,8 +475,8 @@ export async function handleToken(request: Request, env: Env): Promise ApiUseKeyConnector: false, scope: 'api offline_access', unofficialServer: true, - UserDecryptionOptions: buildUserDecryptionOptions(user), - userDecryptionOptions: buildUserDecryptionOptions(user), + UserDecryptionOptions: userDecryptionOptions, + userDecryptionOptions: userDecryptionOptions, }; const baseResponse = jsonResponse(response); diff --git a/src/handlers/sends-private.ts b/src/handlers/sends-private.ts index 337ce0e..ced00e4 100644 --- a/src/handlers/sends-private.ts +++ b/src/handlers/sends-private.ts @@ -97,8 +97,9 @@ export async function handleGetSends(request: Request, env: Env, userId: string) sends = await storage.getAllSends(userId); } + const sendResponses = sends.map(sendToResponse); return jsonResponse({ - data: sends.map(sendToResponse), + data: sendResponses, object: 'list', continuationToken, }); diff --git a/src/handlers/sync.ts b/src/handlers/sync.ts index eec030b..e7b1d31 100644 --- a/src/handlers/sync.ts +++ b/src/handlers/sync.ts @@ -10,87 +10,23 @@ import { buildUserDecryptionOptions, } from '../utils/user-decryption'; -interface SyncCacheEntry { - userId: string; - revisionDate: string; - body: string; - expiresAt: number; - bytes: number; +function buildSyncCacheRequest(request: Request, userId: string, revisionDate: string, excludeDomains: boolean): Request { + const url = new URL(request.url); + const cacheUrl = new URL( + `/__nodewarden/cache/sync/${encodeURIComponent(userId)}/${encodeURIComponent(revisionDate)}/${excludeDomains ? '1' : '0'}`, + url.origin + ); + return new Request(cacheUrl.toString(), { method: 'GET' }); } -const syncResponseCache = new Map(); -let syncResponseCacheTotalBytes = 0; -const textEncoder = new TextEncoder(); - -function buildSyncCacheKey(userId: string, revisionDate: string, excludeDomains: boolean): string { - return `${userId}:${revisionDate}:${excludeDomains ? '1' : '0'}`; -} - -function readSyncCache(key: string): string | null { - const hit = syncResponseCache.get(key); +async function readSyncCache(cacheRequest: Request): Promise { + const hit = await caches.default.match(cacheRequest); if (!hit) return null; - if (hit.expiresAt <= Date.now()) { - deleteSyncCacheEntry(key, hit); - return null; - } - return hit.body; + return new Response(hit.body, hit); } -function deleteSyncCacheEntry(key: string, entry?: SyncCacheEntry): void { - const existing = entry ?? syncResponseCache.get(key); - if (!existing) return; - syncResponseCache.delete(key); - syncResponseCacheTotalBytes = Math.max(0, syncResponseCacheTotalBytes - existing.bytes); -} - -function pruneExpiredSyncCache(nowMs: number = Date.now()): void { - for (const [key, entry] of syncResponseCache.entries()) { - if (entry.expiresAt <= nowMs) { - deleteSyncCacheEntry(key, entry); - } - } -} - -function pruneStaleUserSyncCache(userId: string, revisionDate: string): void { - for (const [key, entry] of syncResponseCache.entries()) { - if (entry.userId === userId && entry.revisionDate !== revisionDate) { - deleteSyncCacheEntry(key, entry); - } - } -} - -function writeSyncCache(userId: string, revisionDate: string, key: string, body: string): void { - const nowMs = Date.now(); - pruneExpiredSyncCache(nowMs); - pruneStaleUserSyncCache(userId, revisionDate); - - const bodyBytes = textEncoder.encode(body).byteLength; - if (bodyBytes > LIMITS.cache.syncResponseMaxBodyBytes) { - return; - } - - const existing = syncResponseCache.get(key); - if (existing) { - deleteSyncCacheEntry(key, existing); - } - - while ( - syncResponseCache.size >= LIMITS.cache.syncResponseMaxEntries || - syncResponseCacheTotalBytes + bodyBytes > LIMITS.cache.syncResponseMaxTotalBytes - ) { - const oldestKey = syncResponseCache.keys().next().value as string | undefined; - if (!oldestKey) break; - deleteSyncCacheEntry(oldestKey); - } - - syncResponseCache.set(key, { - userId, - revisionDate, - body, - expiresAt: nowMs + LIMITS.cache.syncResponseTtlMs, - bytes: bodyBytes, - }); - syncResponseCacheTotalBytes += bodyBytes; +async function writeSyncCache(cacheRequest: Request, response: Response): Promise { + await caches.default.put(cacheRequest, response.clone()); } // GET /api/sync @@ -99,28 +35,28 @@ export async function handleSync(request: Request, env: Env, userId: string): Pr const url = new URL(request.url); const excludeDomainsParam = url.searchParams.get('excludeDomains'); const excludeDomains = excludeDomainsParam !== null && /^(1|true|yes)$/i.test(excludeDomainsParam); - + const user = await storage.getUserById(userId); if (!user) { return errorResponse('User not found', 404); } const revisionDate = await storage.getRevisionDate(userId); - const cacheKey = buildSyncCacheKey(userId, revisionDate, excludeDomains); - const cachedBody = readSyncCache(cacheKey); - if (cachedBody) { - return new Response(cachedBody, { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + const cacheRequest = buildSyncCacheRequest(request, userId, revisionDate, excludeDomains); + const cachedResponse = await readSyncCache(cacheRequest); + if (cachedResponse) { + return cachedResponse; } - const ciphers = await storage.getAllCiphers(userId); - const folders = await storage.getAllFolders(userId); - const sends = await storage.getAllSends(userId); - const attachmentsByCipher = await storage.getAttachmentsByUserId(userId); + const [ciphers, folders, sends, attachmentsByCipher] = await Promise.all([ + storage.getAllCiphers(userId), + storage.getAllFolders(userId), + storage.getAllSends(userId), + storage.getAttachmentsByUserId(userId), + ]); + const accountKeys = buildAccountKeys(user); + const userDecryptionOptions = buildUserDecryptionOptions(user); - // Build profile response const profile: ProfileResponse = { id: user.id, name: user.name, @@ -134,7 +70,7 @@ export async function handleSync(request: Request, env: Env, userId: string): Pr twoFactorEnabled: !!user.totpSecret, key: user.key, privateKey: user.privateKey, - accountKeys: buildAccountKeys(user), + accountKeys, securityStamp: user.securityStamp || user.id, organizations: [], providers: [], @@ -146,23 +82,24 @@ export async function handleSync(request: Request, env: Env, userId: string): Pr object: 'profile', }; - // Build cipher responses with attachments const cipherResponses: CipherResponse[] = []; for (const cipher of ciphers) { - const attachments = attachmentsByCipher.get(cipher.id) || []; - cipherResponses.push(cipherToResponse(cipher, attachments)); + cipherResponses.push(cipherToResponse(cipher, attachmentsByCipher.get(cipher.id) || [])); } - // Build folder responses - const folderResponses: FolderResponse[] = folders.map(folder => ({ - id: folder.id, - name: folder.name, - revisionDate: folder.updatedAt, - object: 'folder', - })); + const folderResponses: FolderResponse[] = []; + for (const folder of folders) { + folderResponses.push({ + id: folder.id, + name: folder.name, + revisionDate: folder.updatedAt, + object: 'folder', + }); + } + const sendResponses = sends.map(sendToResponse); const syncResponse: SyncResponse = { - profile: profile, + profile, folders: folderResponses, collections: [], ciphers: cipherResponses, @@ -174,25 +111,25 @@ export async function handleSync(request: Request, env: Env, userId: string): Pr object: 'domains', }, policies: [], - sends: sends.map(sendToResponse), + sends: sendResponses, UserDecryption: { - MasterPasswordUnlock: buildUserDecryptionOptions(user).MasterPasswordUnlock, + MasterPasswordUnlock: userDecryptionOptions.MasterPasswordUnlock, TrustedDeviceOption: null, KeyConnectorOption: null, Object: 'userDecryption', }, - // PascalCase for desktop/browser clients - UserDecryptionOptions: buildUserDecryptionOptions(user), - // camelCase for Android client (SyncResponseJson uses @SerialName("userDecryption")) + UserDecryptionOptions: userDecryptionOptions, userDecryption: buildUserDecryptionCompat(user) as SyncResponse['userDecryption'], object: 'sync', }; - const body = JSON.stringify(syncResponse); - writeSyncCache(userId, revisionDate, cacheKey, body); - - return new Response(body, { + const response = new Response(JSON.stringify(syncResponse), { status: 200, - headers: { 'Content-Type': 'application/json' }, + headers: { + 'Content-Type': 'application/json', + 'Cache-Control': `private, max-age=${Math.max(1, Math.floor(LIMITS.cache.syncResponseTtlMs / 1000))}`, + }, }); + await writeSyncCache(cacheRequest, response); + return response; } diff --git a/src/services/storage-schema.ts b/src/services/storage-schema.ts index 7b1d947..b29e9b1 100644 --- a/src/services/storage-schema.ts +++ b/src/services/storage-schema.ts @@ -27,6 +27,7 @@ const SCHEMA_STATEMENTS: readonly string[] = [ 'CREATE INDEX IF NOT EXISTS idx_ciphers_user_updated ON ciphers(user_id, updated_at)', 'CREATE INDEX IF NOT EXISTS idx_ciphers_user_archived ON ciphers(user_id, archived_at)', 'CREATE INDEX IF NOT EXISTS idx_ciphers_user_deleted ON ciphers(user_id, deleted_at)', + 'CREATE INDEX IF NOT EXISTS idx_ciphers_user_deleted_updated ON ciphers(user_id, deleted_at, updated_at)', 'CREATE TABLE IF NOT EXISTS folders (' + 'id TEXT PRIMARY KEY, user_id TEXT NOT NULL, name TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, ' + @@ -47,6 +48,7 @@ const SCHEMA_STATEMENTS: readonly string[] = [ 'FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE)', 'CREATE INDEX IF NOT EXISTS idx_sends_user_updated ON sends(user_id, updated_at)', 'CREATE INDEX IF NOT EXISTS idx_sends_user_deletion ON sends(user_id, deletion_date)', + 'CREATE INDEX IF NOT EXISTS idx_sends_user_updated_id ON sends(user_id, updated_at, id)', 'ALTER TABLE sends ADD COLUMN auth_type INTEGER NOT NULL DEFAULT 2', 'ALTER TABLE sends ADD COLUMN emails TEXT', diff --git a/webapp/src/lib/api/send.ts b/webapp/src/lib/api/send.ts index 65196e0..cc7d02d 100644 --- a/webapp/src/lib/api/send.ts +++ b/webapp/src/lib/api/send.ts @@ -1,6 +1,7 @@ import { base64ToBytes, bytesToBase64, decryptBw, decryptBwFileData, decryptStr, encryptBw, encryptBwFileData, hkdf, pbkdf2 } from '../crypto'; import type { Send, SendDraft, SessionState } from '../types'; import { chunkArray, createApiError, parseErrorMessage, parseJson, uploadDirectEncryptedPayload, type AuthedFetch } from './shared'; +import { loadVaultSyncSnapshot } from './vault-sync'; function toIsoDateFromDays(value: string, required: boolean): string | null { const raw = String(value || '').trim(); @@ -61,10 +62,8 @@ function parseMaxAccessCountRaw(value: string): number | null { } export async function getSends(authedFetch: AuthedFetch): Promise { - const resp = await authedFetch('/api/sends'); - if (!resp.ok) throw new Error('Failed to load sends'); - const body = await parseJson<{ object: 'list'; data: Send[] }>(resp); - return body?.data || []; + const body = await loadVaultSyncSnapshot(authedFetch); + return body.sends || []; } export async function createSend( diff --git a/webapp/src/lib/api/vault-sync.ts b/webapp/src/lib/api/vault-sync.ts new file mode 100644 index 0000000..4dc2474 --- /dev/null +++ b/webapp/src/lib/api/vault-sync.ts @@ -0,0 +1,31 @@ +import type { Cipher, Folder, Send } from '../types'; +import { parseJson, type AuthedFetch } from './shared'; + +interface VaultSyncResponse { + ciphers?: Cipher[]; + folders?: Folder[]; + sends?: Send[]; +} + +const pendingSyncRequests = new WeakMap>(); + +export async function loadVaultSyncSnapshot(authedFetch: AuthedFetch): Promise { + const existing = pendingSyncRequests.get(authedFetch); + if (existing) return existing; + + const request = (async () => { + const resp = await authedFetch('/api/sync'); + if (!resp.ok) throw new Error('Failed to load vault'); + const body = await parseJson(resp); + return body || {}; + })(); + + pendingSyncRequests.set(authedFetch, request); + try { + return await request; + } finally { + if (pendingSyncRequests.get(authedFetch) === request) { + pendingSyncRequests.delete(authedFetch); + } + } +} diff --git a/webapp/src/lib/api/vault.ts b/webapp/src/lib/api/vault.ts index 56b80b3..ad06dea 100644 --- a/webapp/src/lib/api/vault.ts +++ b/webapp/src/lib/api/vault.ts @@ -2,7 +2,6 @@ import { base64ToBytes, decryptBw, decryptBwFileData, decryptStr, encryptBw, enc import type { Cipher, Folder, - ListResponse, SessionState, VaultDraft, VaultDraftField, @@ -16,12 +15,11 @@ import { type AuthedFetch, } from './shared'; import { readResponseBytesWithProgress } from '../download'; +import { loadVaultSyncSnapshot } from './vault-sync'; export async function getFolders(authedFetch: AuthedFetch): Promise { - const resp = await authedFetch('/api/folders'); - if (!resp.ok) throw new Error('Failed to load folders'); - const body = await parseJson>(resp); - return body?.data || []; + const body = await loadVaultSyncSnapshot(authedFetch); + return body.folders || []; } export async function createFolder( @@ -93,10 +91,8 @@ export async function updateFolder( } export async function getCiphers(authedFetch: AuthedFetch): Promise { - const resp = await authedFetch('/api/ciphers?deleted=true'); - if (!resp.ok) throw new Error('Failed to load ciphers'); - const body = await parseJson>(resp); - return body?.data || []; + const body = await loadVaultSyncSnapshot(authedFetch); + return body.ciphers || []; } export interface CiphersImportPayload {