feat: enhance backup process with lease management and attachment deletion

- Implemented a backup runner lease mechanism to prevent concurrent backup executions.
- Added `deleteAllAttachmentsForCiphers` function to delete attachments for multiple ciphers efficiently.
- Introduced `bulkDeleteAttachmentsByIds` method in storage to handle batch deletion of attachments.
- Updated backup execution logic to utilize the new lease management and ensure timely updates during the backup process.
- Refactored cipher deletion to handle attachments more effectively.
- Improved website icon loading with a dedicated caching mechanism for better performance.
- Added new index on `ciphers` table for `folder_id` to optimize queries related to folder management.
- Enhanced response handling for CORS policy to allow credentials for specific origins.
This commit is contained in:
shuaiplus
2026-04-28 23:40:43 +08:00
parent 69b98f9e67
commit 68ded534a4
16 changed files with 505 additions and 284 deletions
+16 -3
View File
@@ -445,12 +445,25 @@ export async function handleDeleteAttachment(
export async function deleteAllAttachmentsForCipher(
env: Env,
cipherId: string
): Promise<void> {
await deleteAllAttachmentsForCiphers(env, [cipherId]);
}
export async function deleteAllAttachmentsForCiphers(
env: Env,
cipherIds: string[]
): Promise<void> {
const storage = new StorageService(env.DB);
const attachments = await storage.getAttachmentsByCipher(cipherId);
await runWithConcurrency(attachments, LIMITS.performance.attachmentDeleteConcurrency, async (attachment) => {
const attachmentsByCipher = await storage.getAttachmentsByCipherIds(cipherIds);
const attachments = Array.from(attachmentsByCipher.entries()).flatMap(([ownedCipherId, items]) =>
items.map((attachment) => ({ attachment, cipherId: ownedCipherId }))
);
if (!attachments.length) return;
await runWithConcurrency(attachments, LIMITS.performance.attachmentDeleteConcurrency, async ({ attachment, cipherId }) => {
const path = getAttachmentObjectKey(cipherId, attachment.id);
await deleteBlobObject(env, path);
await storage.deleteAttachment(attachment.id);
});
await storage.bulkDeleteAttachmentsByIds(attachments.map(({ attachment }) => attachment.id));
}
+156 -15
View File
@@ -14,6 +14,7 @@ import {
getBackupLocalDateKey,
getDefaultBackupSettings,
getBackupSettingsRepairState,
hasBackupSlotBetween,
isBackupDueNow,
loadBackupSettings,
normalizeBackupSettingsInput,
@@ -80,6 +81,98 @@ function getBackupDestinationSummary(destination: BackupDestinationRecord | null
};
}
const BACKUP_RUNNER_LOCK_KEY = 'backup.runner.lock.v1';
const BACKUP_RUNNER_LEASE_MS = 10 * 60 * 1000;
const BACKUP_RUNNER_HEARTBEAT_MS = 30 * 1000;
interface BackupRunnerLease {
token: string;
touch: () => Promise<void>;
release: () => Promise<void>;
}
async function acquireBackupRunnerLease(env: Env, reason: string): Promise<BackupRunnerLease | null> {
const token = generateUUID();
const nowMs = Date.now();
const expiresAtMs = nowMs + BACKUP_RUNNER_LEASE_MS;
const value = JSON.stringify({
token,
reason,
acquiredAt: new Date(nowMs).toISOString(),
touchedAt: new Date(nowMs).toISOString(),
expiresAtMs,
});
const result = await env.DB
.prepare(
`INSERT INTO config(key, value) VALUES(?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
WHERE COALESCE(CAST(json_extract(config.value, '$.expiresAtMs') AS INTEGER), 0) <= ?`
)
.bind(BACKUP_RUNNER_LOCK_KEY, value, nowMs)
.run();
if ((result.meta?.changes || 0) < 1) {
return null;
}
return {
token,
touch: async () => {
const nextNowMs = Date.now();
const nextValue = JSON.stringify({
token,
reason,
acquiredAt: new Date(nowMs).toISOString(),
touchedAt: new Date(nextNowMs).toISOString(),
expiresAtMs: nextNowMs + BACKUP_RUNNER_LEASE_MS,
});
await env.DB
.prepare(
`UPDATE config
SET value = ?
WHERE key = ?
AND json_extract(value, '$.token') = ?`
)
.bind(nextValue, BACKUP_RUNNER_LOCK_KEY, token)
.run();
},
release: async () => {
await env.DB
.prepare(
`DELETE FROM config
WHERE key = ?
AND json_extract(value, '$.token') = ?`
)
.bind(BACKUP_RUNNER_LOCK_KEY, token)
.run();
},
};
}
async function withBackupRunnerLease<T>(
env: Env,
reason: string,
task: (keepAlive: () => Promise<void>) => Promise<T>
): Promise<T | null> {
const lease = await acquireBackupRunnerLease(env, reason);
if (!lease) return null;
let lastHeartbeatAt = 0;
const keepAlive = async () => {
const nowMs = Date.now();
if (nowMs - lastHeartbeatAt < BACKUP_RUNNER_HEARTBEAT_MS) return;
lastHeartbeatAt = nowMs;
await lease.touch();
};
try {
await keepAlive();
return await task(keepAlive);
} finally {
await lease.release();
}
}
function ensureBackupBlobName(value: string): string {
const normalized = String(value || '').trim().replace(/\\/g, '/').replace(/^\/+|\/+$/g, '');
if (!normalized) {
@@ -160,6 +253,7 @@ async function executeConfiguredBackup(
actorUserId: string | null,
trigger: 'manual' | 'scheduled',
destinationId?: string | null,
keepAlive?: (() => Promise<void>) | null,
progress?: ((event: {
operation: 'backup-remote-run';
step: string;
@@ -172,6 +266,9 @@ async function executeConfiguredBackup(
}) => Promise<void>) | null
): Promise<{ fileName: string; fileSize: number; remotePath: string; provider: string }> {
const maxArchiveUploadAttempts = 3;
const touchLease = async () => {
await keepAlive?.();
};
const currentSettings = await loadBackupSettings(storage, env, 'UTC');
const destination = requireBackupDestination(currentSettings, destinationId);
@@ -180,9 +277,11 @@ async function executeConfiguredBackup(
destination.runtime.lastAttemptLocalDate = getBackupLocalDateKey(now, destination.schedule.timezone);
destination.runtime.lastErrorAt = null;
destination.runtime.lastErrorMessage = null;
await touchLease();
await saveBackupSettings(storage, env, currentSettings);
try {
await touchLease();
await progress?.({
operation: 'backup-remote-run',
step: 'remote_run_prepare',
@@ -190,6 +289,7 @@ async function executeConfiguredBackup(
stageTitle: 'txt_backup_remote_run_progress_prepare_title',
stageDetail: 'txt_backup_remote_run_progress_prepare_detail',
});
await touchLease();
const archive = await buildBackupArchive(env, now, {
includeAttachments: destination.includeAttachments,
timeZone: destination.schedule.timezone,
@@ -219,9 +319,11 @@ async function executeConfiguredBackup(
});
const remoteSession = createRemoteBackupTransferSession(destination);
if (destination.includeAttachments) {
await touchLease();
const remoteAttachmentIndex = await loadRemoteAttachmentIndex(remoteSession);
let attachmentIndexChanged = false;
for (const attachment of archive.manifest.attachmentBlobs || []) {
await touchLease();
if (remoteAttachmentIndex.get(attachment.blobName) === attachment.sizeBytes) {
continue;
}
@@ -238,11 +340,13 @@ async function executeConfiguredBackup(
attachmentIndexChanged = true;
}
if (attachmentIndexChanged) {
await touchLease();
await saveRemoteAttachmentIndex(remoteSession, remoteAttachmentIndex);
}
}
let upload: Awaited<ReturnType<typeof uploadBackupArchive>> | null = null;
for (let attempt = 1; attempt <= maxArchiveUploadAttempts; attempt++) {
await touchLease();
await progress?.({
operation: 'backup-remote-run',
step: 'remote_run_upload_archive',
@@ -252,6 +356,7 @@ async function executeConfiguredBackup(
});
upload = await remoteSession.uploadArchive(archive.bytes, archive.fileName);
try {
await touchLease();
await progress?.({
operation: 'backup-remote-run',
step: 'remote_run_verify_archive',
@@ -282,6 +387,7 @@ async function executeConfiguredBackup(
let prunedFileCount = 0;
let pruneErrorMessage: string | null = null;
try {
await touchLease();
await progress?.({
operation: 'backup-remote-run',
step: 'remote_run_cleanup',
@@ -300,8 +406,10 @@ async function executeConfiguredBackup(
destination.runtime.lastUploadedFileName = archive.fileName;
destination.runtime.lastUploadedSizeBytes = archive.bytes.byteLength;
destination.runtime.lastUploadedDestination = upload.remotePath;
await touchLease();
await saveBackupSettings(storage, env, currentSettings);
await touchLease();
await writeAuditLog(storage, actorUserId, `admin.backup.remote.${trigger}`, 'backup', null, {
...getBackupDestinationSummary(destination),
provider: upload.provider,
@@ -332,8 +440,10 @@ async function executeConfiguredBackup(
} catch (error) {
destination.runtime.lastErrorAt = new Date().toISOString();
destination.runtime.lastErrorMessage = error instanceof Error ? error.message : 'Backup upload failed';
await touchLease();
await saveBackupSettings(storage, env, currentSettings);
await touchLease();
await writeAuditLog(storage, actorUserId, `admin.backup.remote.${trigger}.failed`, 'backup', null, {
...getBackupDestinationSummary(destination),
error: destination.runtime.lastErrorMessage,
@@ -404,13 +514,30 @@ async function runImportAndAudit(
}
export async function runScheduledBackupIfDue(env: Env): Promise<void> {
const storage = new StorageService(env.DB);
const settings = await loadBackupSettings(storage, env, 'UTC');
const now = new Date();
for (const destination of settings.destinations) {
if (!isBackupDueNow(destination, now, BACKUP_SCHEDULER_WINDOW_MINUTES)) continue;
await executeConfiguredBackup(env, storage, null, 'scheduled', destination.id);
}
await withBackupRunnerLease(env, 'scheduled', async (keepAlive) => {
const storage = new StorageService(env.DB);
let scanStartMs = Date.now();
while (true) {
await keepAlive();
const settings = await loadBackupSettings(storage, env, 'UTC');
const now = new Date();
const dueDestinations = settings.destinations.filter((destination) =>
isBackupDueNow(destination, now, BACKUP_SCHEDULER_WINDOW_MINUTES)
|| hasBackupSlotBetween(destination, new Date(scanStartMs), now)
);
if (!dueDestinations.length) {
return;
}
scanStartMs = now.getTime();
for (const destination of dueDestinations) {
await keepAlive();
await executeConfiguredBackup(env, storage, null, 'scheduled', destination.id, keepAlive);
}
}
});
}
export async function handleGetAdminBackupSettings(request: Request, env: Env, actorUser: User): Promise<Response> {
@@ -512,7 +639,6 @@ export async function handleRepairAdminBackupSettings(request: Request, env: Env
export async function handleRunAdminConfiguredBackup(request: Request, env: Env, actorUser: User): Promise<Response> {
if (!isAdmin(actorUser)) return errorResponse('Forbidden', 403);
const storage = new StorageService(env.DB);
try {
let body: { destinationId?: string } | null = null;
try {
@@ -536,17 +662,32 @@ export async function handleRunAdminConfiguredBackup(request: Request, env: Env,
}) => {
await notifyUserBackupProgress(env, actorUser.id, event, targetDeviceIdentifier);
};
const result = await executeConfiguredBackup(env, storage, actorUser.id, 'manual', body?.destinationId || null, progress);
const settings = await loadBackupSettings(storage, env, 'UTC');
const outcome = await withBackupRunnerLease(env, `manual:${actorUser.id}`, async (keepAlive) => {
const storage = new StorageService(env.DB);
const result = await executeConfiguredBackup(
env,
storage,
actorUser.id,
'manual',
body?.destinationId || null,
keepAlive,
progress
);
const settings = await loadBackupSettings(storage, env, 'UTC');
return { result, settings };
});
if (!outcome) {
return errorResponse('Another backup run is already in progress', 409);
}
return jsonResponse({
object: 'backup-run',
result: {
fileName: result.fileName,
fileSize: result.fileSize,
provider: result.provider,
remotePath: result.remotePath,
fileName: outcome.result.fileName,
fileSize: outcome.result.fileSize,
provider: outcome.result.provider,
remotePath: outcome.result.remotePath,
},
settings,
settings: outcome.settings,
});
} catch (error) {
return errorResponse(error instanceof Error ? error.message : 'Backup run failed', 500);
+8 -4
View File
@@ -14,7 +14,7 @@ import { StorageService } from '../services/storage';
import { notifyUserVaultSync } from '../durable/notifications-hub';
import { jsonResponse, errorResponse } from '../utils/response';
import { generateUUID } from '../utils/uuid';
import { deleteAllAttachmentsForCipher } from './attachments';
import { deleteAllAttachmentsForCipher, deleteAllAttachmentsForCiphers } from './attachments';
import { parsePagination, encodeContinuationToken } from '../utils/pagination';
import { readActingDeviceIdentifier } from '../utils/device';
@@ -744,11 +744,15 @@ export async function handleBulkPermanentDeleteCiphers(request: Request, env: En
return new Response(null, { status: 204 });
}
for (const id of ids) {
await deleteAllAttachmentsForCipher(env, id);
const ownedCiphers = await storage.getCiphersByIds(ids, userId);
const ownedIds = ownedCiphers.map((cipher) => cipher.id);
if (!ownedIds.length) {
return new Response(null, { status: 204 });
}
const revisionDate = await storage.bulkDeleteCiphers(ids, userId);
await deleteAllAttachmentsForCiphers(env, ownedIds);
const revisionDate = await storage.bulkDeleteCiphers(ownedIds, userId);
if (revisionDate) {
notifyVaultSyncForRequest(request, env, userId, revisionDate);
}
+2 -2
View File
@@ -61,7 +61,7 @@ function handleNwFavicon(): Response {
status: 200,
headers: {
'Content-Type': 'image/svg+xml; charset=utf-8',
'Cache-Control': `public, max-age=${LIMITS.cache.iconTtlSeconds}`,
'Cache-Control': `public, max-age=${LIMITS.cache.iconTtlSeconds}, immutable`,
},
});
}
@@ -181,7 +181,7 @@ async function handleWebsiteIcon(host: string, fallbackMode: 'default' | 'not-fo
status: 200,
headers: {
'Content-Type': resp.headers.get('Content-Type') || 'image/png',
'Cache-Control': `public, max-age=${LIMITS.cache.iconTtlSeconds}`,
'Cache-Control': `public, max-age=${LIMITS.cache.iconTtlSeconds}, immutable`,
},
});
}
+44
View File
@@ -598,6 +598,50 @@ function getBackupSlotStartsForLocalDay(
return slots;
}
export function hasBackupSlotBetween(
destination: BackupDestinationRecord,
startInclusive: Date,
endExclusive: Date
): boolean {
if (!destination.schedule.enabled) return false;
const startMs = startInclusive.getTime();
const endMs = endExclusive.getTime();
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs <= startMs) return false;
const lastAttemptAt = destination.runtime.lastAttemptAt ? new Date(destination.runtime.lastAttemptAt) : null;
const lastAttemptMs = lastAttemptAt && Number.isFinite(lastAttemptAt.getTime())
? lastAttemptAt.getTime()
: Number.NEGATIVE_INFINITY;
const dayCursor = new Date(startMs);
dayCursor.setUTCHours(0, 0, 0, 0);
const endDay = new Date(endMs);
endDay.setUTCHours(0, 0, 0, 0);
const checkedLocalDateKeys = new Set<string>();
while (dayCursor.getTime() <= endDay.getTime() + 24 * 60 * 60 * 1000) {
const localDateKey = getBackupLocalDateKey(dayCursor, destination.schedule.timezone);
if (!checkedLocalDateKeys.has(localDateKey)) {
checkedLocalDateKeys.add(localDateKey);
const slotStarts = getBackupSlotStartsForLocalDay(
localDateKey,
destination.schedule.timezone,
destination.schedule.startTime,
destination.schedule.intervalHours
);
for (const slotStart of slotStarts) {
const slotStartMs = slotStart.getTime();
if (slotStartMs < startMs || slotStartMs >= endMs) continue;
if (lastAttemptMs >= slotStartMs) continue;
return true;
}
}
dayCursor.setUTCDate(dayCursor.getUTCDate() + 1);
}
return false;
}
export function isBackupDueNow(
destination: BackupDestinationRecord,
now: Date,
+16
View File
@@ -34,6 +34,22 @@ export async function deleteAttachment(db: D1Database, id: string): Promise<void
await db.prepare('DELETE FROM attachments WHERE id = ?').bind(id).run();
}
export async function bulkDeleteAttachmentsByIds(
db: D1Database,
sqlChunkSize: SqlChunkSize,
attachmentIds: string[]
): Promise<void> {
const uniqueIds = [...new Set(attachmentIds.map((id) => String(id || '').trim()).filter(Boolean))];
if (!uniqueIds.length) return;
const chunkSize = sqlChunkSize(0);
for (let i = 0; i < uniqueIds.length; i += chunkSize) {
const chunk = uniqueIds.slice(i, i + chunkSize);
const placeholders = chunk.map(() => '?').join(',');
await db.prepare(`DELETE FROM attachments WHERE id IN (${placeholders})`).bind(...chunk).run();
}
}
export async function getAttachmentsByCipher(db: D1Database, cipherId: string): Promise<Attachment[]> {
const res = await db
.prepare('SELECT id, cipher_id, file_name, size, size_name, key FROM attachments WHERE cipher_id = ?')
+21 -37
View File
@@ -1,4 +1,4 @@
import type { Cipher, Folder } from '../types';
import type { Folder } from '../types';
function mapFolderRow(row: any): Folder {
return {
@@ -36,26 +36,18 @@ export async function deleteFolder(db: D1Database, id: string, userId: string):
export async function clearFolderFromCiphers(
db: D1Database,
userId: string,
folderId: string,
saveCipher: (cipher: Cipher) => Promise<void>
folderId: string
): Promise<void> {
const now = new Date().toISOString();
const res = await db
.prepare('SELECT data FROM ciphers WHERE user_id = ? AND folder_id = ?')
.bind(userId, folderId)
.all<{ data: string }>();
for (const row of (res.results || [])) {
let cipher: Cipher;
try {
cipher = JSON.parse(row.data) as Cipher;
} catch {
continue;
}
cipher.folderId = null;
cipher.updatedAt = now;
await saveCipher(cipher);
}
const patch = JSON.stringify({ folderId: null, updatedAt: now });
await db
.prepare(
`UPDATE ciphers
SET folder_id = NULL, updated_at = ?, data = json_patch(data, ?)
WHERE user_id = ? AND folder_id = ?`
)
.bind(now, patch, userId, folderId)
.run();
}
export async function bulkDeleteFolders(
@@ -63,34 +55,26 @@ export async function bulkDeleteFolders(
userId: string,
ids: string[],
sqlChunkSize: (fixedBindCount: number) => number,
saveCipher: (cipher: Cipher) => Promise<void>,
updateRevisionDate: (userId: string) => Promise<string>
): Promise<string | null> {
const uniqueIds = Array.from(new Set(ids.map((id) => String(id || '').trim()).filter(Boolean)));
if (!uniqueIds.length) return null;
const chunkSize = sqlChunkSize(1);
const now = new Date().toISOString();
const patch = JSON.stringify({ folderId: null, updatedAt: now });
const chunkSize = sqlChunkSize(3);
for (let i = 0; i < uniqueIds.length; i += chunkSize) {
const chunk = uniqueIds.slice(i, i + chunkSize);
const placeholders = chunk.map(() => '?').join(',');
const res = await db
.prepare(`SELECT data FROM ciphers WHERE user_id = ? AND folder_id IN (${placeholders})`)
.bind(userId, ...chunk)
.all<{ data: string }>();
for (const row of res.results || []) {
let cipher: Cipher;
try {
cipher = JSON.parse(row.data) as Cipher;
} catch {
continue;
}
cipher.folderId = null;
cipher.updatedAt = now;
await saveCipher(cipher);
}
await db
.prepare(
`UPDATE ciphers
SET folder_id = NULL, updated_at = ?, data = json_patch(data, ?)
WHERE user_id = ? AND folder_id IN (${placeholders})`
)
.bind(now, patch, userId, ...chunk)
.run();
await db
.prepare(`DELETE FROM folders WHERE user_id = ? AND id IN (${placeholders})`)
+1
View File
@@ -29,6 +29,7 @@ const SCHEMA_STATEMENTS: readonly string[] = [
'CREATE INDEX IF NOT EXISTS idx_ciphers_user_archived ON ciphers(user_id, archived_at)',
'CREATE INDEX IF NOT EXISTS idx_ciphers_user_deleted ON ciphers(user_id, deleted_at)',
'CREATE INDEX IF NOT EXISTS idx_ciphers_user_deleted_updated ON ciphers(user_id, deleted_at, updated_at)',
'CREATE INDEX IF NOT EXISTS idx_ciphers_user_folder ON ciphers(user_id, folder_id)',
'CREATE TABLE IF NOT EXISTS folders (' +
'id TEXT PRIMARY KEY, user_id TEXT NOT NULL, name TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, ' +
+7 -3
View File
@@ -51,6 +51,7 @@ import {
} from './storage-cipher-repo';
import {
addAttachmentToCipher as attachStoredAttachmentToCipher,
bulkDeleteAttachmentsByIds as deleteStoredAttachmentsByIds,
deleteAllAttachmentsByCipher as deleteStoredAttachmentsByCipher,
deleteAttachment as deleteStoredAttachment,
getAttachment as findStoredAttachment,
@@ -107,7 +108,7 @@ import {
const TWO_FACTOR_REMEMBER_TTL_MS = 30 * 24 * 60 * 60 * 1000;
const STORAGE_SCHEMA_VERSION_KEY = 'schema.version';
const STORAGE_SCHEMA_VERSION = '2026-04-22';
const STORAGE_SCHEMA_VERSION = '2026-04-28';
// D1-backed storage.
// Contract:
@@ -339,7 +340,6 @@ export class StorageService {
userId,
ids,
this.sqlChunkSize.bind(this),
this.saveCipher.bind(this),
this.updateRevisionDate.bind(this)
);
}
@@ -347,7 +347,7 @@ export class StorageService {
// Clear folder references from all ciphers owned by the user.
// Without this, deleting a folder leaves stale folderId values in cipher JSON.
async clearFolderFromCiphers(userId: string, folderId: string): Promise<void> {
await clearStoredFolderFromCiphers(this.db, userId, folderId, this.saveCipher.bind(this));
await clearStoredFolderFromCiphers(this.db, userId, folderId);
}
async getAllFolders(userId: string): Promise<Folder[]> {
@@ -372,6 +372,10 @@ export class StorageService {
await deleteStoredAttachment(this.db, id);
}
async bulkDeleteAttachmentsByIds(ids: string[]): Promise<void> {
await deleteStoredAttachmentsByIds(this.db, this.sqlChunkSize.bind(this), ids);
}
async getAttachmentsByCipher(cipherId: string): Promise<Attachment[]> {
return listStoredAttachmentsByCipher(this.db, cipherId);
}
+1 -1
View File
@@ -48,7 +48,7 @@ function getCorsPolicy(request: Request): { allowOrigin: string | null; allowCre
return { allowOrigin: origin, allowCredentials: true };
}
if (isExtensionOrigin(origin)) {
return { allowOrigin: origin, allowCredentials: false };
return { allowOrigin: origin, allowCredentials: true };
}
return { allowOrigin: null, allowCredentials: false };
}