From 5ed7c949c164e5bc5bb2a2c633d90fdaef16031a Mon Sep 17 00:00:00 2001 From: shuaiplus <2327005759@qq.com> Date: Sun, 7 Jun 2026 21:06:34 +0800 Subject: [PATCH] feat: add remote backup restore and attachment download functionality --- src/durable/backup-transfer-runner.ts | 178 +++++++++++++++- src/handlers/backup.ts | 291 ++++++++++++++++++++------ 2 files changed, 400 insertions(+), 69 deletions(-) diff --git a/src/durable/backup-transfer-runner.ts b/src/durable/backup-transfer-runner.ts index b0b099c..5c1d9ee 100644 --- a/src/durable/backup-transfer-runner.ts +++ b/src/durable/backup-transfer-runner.ts @@ -2,15 +2,25 @@ import type { Env } from '../types'; import type { BackupDestinationRecord } from '../services/backup-config'; import { BACKUP_SCHEDULER_WINDOW_MINUTES, + requireBackupDestination, hasBackupSlotBetween, isBackupDueNow, loadBackupSettings, } from '../services/backup-config'; -import { createRemoteBackupTransferSession } from '../services/backup-uploader'; +import { + createRemoteBackupTransferSession, + downloadRemoteBackupFile, + ensureRemoteRestoreCandidate, +} from '../services/backup-uploader'; import { getBlobObject } from '../services/blob-store'; import { StorageService } from '../services/storage'; -import { notifyUserBackupProgress } from './notifications-hub'; -import { executeConfiguredBackup } from '../handlers/backup'; +import { notifyUserBackupProgress, notifyUserBackupRestoreProgress } from './notifications-hub'; +import { + executeConfiguredBackup, + importAndAuditRemoteBackupFile, +} from '../handlers/backup'; +import { verifyBackupArchiveFileNameChecksum } from '../services/backup-archive'; +import { zipSync } from 'fflate'; const BACKUP_JOB_STATE_KEY = 'backup.job.state.v1'; const BACKUP_JOB_LEASE_MS = 10 * 60 * 1000; @@ -31,6 +41,16 @@ interface RemoteAttachmentChunkRequest { }>; } +interface RemoteAttachmentDownloadRequest { + destination: BackupDestinationRecord; + blobName?: string | null; +} + +interface RemoteAttachmentBatchDownloadRequest { + destination: BackupDestinationRecord; + blobNames?: string[] | null; +} + interface ConfiguredBackupRunRequest { actorUserId?: string | null; auditMetadata?: Record | null; @@ -39,6 +59,16 @@ interface ConfiguredBackupRunRequest { trigger?: 'manual' | 'scheduled'; } +interface RemoteBackupRestoreRequest { + actorUserId?: string | null; + allowChecksumMismatch?: boolean; + auditMetadata?: Record | null; + destinationId?: string | null; + path?: string | null; + replaceExisting?: boolean; + targetDeviceIdentifier?: string | null; +} + function badRequest(message: string, status: number = 400): Response { return new Response(JSON.stringify({ error: message }), { status, @@ -229,6 +259,82 @@ export class BackupTransferRunner { } } + private async restoreRemoteBackup(request: Request): Promise { + let body: RemoteBackupRestoreRequest; + try { + body = await request.json(); + } catch { + return badRequest('Remote restore payload is invalid'); + } + + const actorUserId = String(body.actorUserId || '').trim() || null; + if (!actorUserId) { + return badRequest('Remote restore requires an actor'); + } + + const token = await this.acquireJob(`restore:${actorUserId}`); + if (!token) { + return badRequest('Another backup or restore run is already in progress', 409); + } + + try { + await this.touchJob(token); + const storage = new StorageService(this.env.DB); + const settings = await loadBackupSettings(storage, this.env, 'UTC'); + const destination = requireBackupDestination(settings, body.destinationId || null); + const path = ensureRemoteRestoreCandidate(String(body.path || '')); + const restoreFileNameFromPath = path.split('/').pop() || path; + const targetDeviceIdentifier = String(body.targetDeviceIdentifier || '').trim() || null; + const replaceExisting = !!body.replaceExisting; + + await notifyUserBackupRestoreProgress( + this.env, + actorUserId, + { + operation: 'backup-restore', + source: 'remote', + step: 'remote_fetch_archive', + fileName: restoreFileNameFromPath, + stageTitle: 'txt_backup_restore_progress_remote_fetch_title', + stageDetail: 'txt_backup_restore_progress_remote_fetch_detail', + replaceExisting, + }, + targetDeviceIdentifier + ); + + const remoteFile = await downloadRemoteBackupFile(destination, path); + const checksumOk = await verifyBackupArchiveFileNameChecksum(remoteFile.bytes, remoteFile.fileName || path); + if (!checksumOk && !body.allowChecksumMismatch) { + return badRequest('Remote backup file checksum does not match its filename'); + } + + const result = await importAndAuditRemoteBackupFile( + this.env, + storage, + actorUserId, + remoteFile, + destination, + path, + replaceExisting, + !checksumOk, + body.auditMetadata || null, + targetDeviceIdentifier + ); + + return new Response(JSON.stringify(result.result), { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + 'Cache-Control': 'no-store', + }, + }); + } catch (error) { + return badRequest(error instanceof Error ? error.message : 'Remote backup restore failed', 500); + } finally { + await this.releaseJob(token); + } + } + async fetch(request: Request): Promise { const url = new URL(request.url); if (request.method !== 'POST') { @@ -243,6 +349,72 @@ export class BackupTransferRunner { return this.runScheduledBackups(); } + if (url.pathname === '/internal/restore-remote-backup') { + return this.restoreRemoteBackup(request); + } + + if (url.pathname === '/internal/download-remote-attachment') { + let body: RemoteAttachmentDownloadRequest; + try { + body = await request.json(); + } catch { + return badRequest('Remote attachment download payload is invalid'); + } + const blobName = String(body?.blobName || '').trim(); + if (!body?.destination || !blobName) { + return badRequest('Remote attachment download payload is invalid'); + } + const file = await downloadRemoteBackupFile(body.destination, `attachments/${blobName}`).catch(() => null); + if (!file) { + return badRequest('Remote attachment not found', 404); + } + return new Response(file.bytes, { + status: 200, + headers: { + 'Content-Type': file.contentType || 'application/octet-stream', + 'Cache-Control': 'no-store', + }, + }); + } + + if (url.pathname === '/internal/download-remote-attachment-batch') { + let body: RemoteAttachmentBatchDownloadRequest; + try { + body = await request.json(); + } catch { + return badRequest('Remote attachment batch download payload is invalid'); + } + const blobNames = Array.from(new Set( + (Array.isArray(body?.blobNames) ? body.blobNames : []) + .map((blobName) => String(blobName || '').trim()) + .filter(Boolean) + )); + if (!body?.destination || !blobNames.length || blobNames.length > 40) { + return badRequest('Remote attachment batch download payload is invalid'); + } + + const encoder = new TextEncoder(); + const entries: Array<{ blobName: string; path: string }> = []; + const files: Record = {}; + for (let i = 0; i < blobNames.length; i += 1) { + const blobName = blobNames[i]; + const file = await downloadRemoteBackupFile(body.destination, `attachments/${blobName}`).catch(() => null); + if (!file) continue; + const path = `files/${i}.bin`; + entries.push({ blobName, path }); + files[path] = file.bytes; + } + files['manifest.json'] = encoder.encode(JSON.stringify({ version: 1, entries })); + + return new Response(zipSync(files), { + status: 200, + headers: { + 'Content-Type': 'application/zip', + 'Cache-Control': 'no-store', + }, + }); + } + if (url.pathname !== '/internal/upload-attachment-chunk') { return badRequest('Not found', 404); } diff --git a/src/handlers/backup.ts b/src/handlers/backup.ts index db600bf..288eb65 100644 --- a/src/handlers/backup.ts +++ b/src/handlers/backup.ts @@ -4,6 +4,7 @@ import { type BackupArchiveBundle, buildBackupArchive, inspectBackupArchiveFileNameChecksum, + parseBackupArchive, verifyBackupArchiveFileNameChecksum, } from '../services/backup-archive'; import { @@ -29,6 +30,7 @@ import { } from '../services/backup-import'; import { type RemoteBackupTransferSession, + type RemoteBackupFile, createRemoteBackupTransferSession, deleteRemoteBackupFile, downloadRemoteBackupFile, @@ -41,6 +43,7 @@ import { StorageService } from '../services/storage'; import { auditRequestMetadata, writeAuditEvent } from '../services/audit-events'; import { getBlobObject } from '../services/blob-store'; import { notifyUserBackupProgress, notifyUserBackupRestoreProgress } from '../durable/notifications-hub'; +import { unzipSync } from 'fflate'; function isAdmin(user: User): boolean { return user.role === 'admin' && user.status === 'active'; @@ -107,6 +110,7 @@ const REMOTE_ATTACHMENT_SYNC_EXTERNAL_SUBREQUEST_LIMIT = 50; const REMOTE_ATTACHMENT_SYNC_SUBREQUEST_RESERVE = 6; const REMOTE_ATTACHMENT_SYNC_MAX_WEB_DAV_BATCH_SIZE = 18; const REMOTE_ATTACHMENT_SYNC_MAX_S3_BATCH_SIZE = 40; +const REMOTE_ATTACHMENT_RESTORE_BATCH_SIZE = 40; function countRemotePathSegments(value: string): number { return String(value || '').replace(/\\/g, '/').split('/').filter(Boolean).length; @@ -503,14 +507,223 @@ async function runScheduledBackupsInDurableObject(env: Env): Promise { } } +async function downloadRemoteAttachmentViaDurableObject( + env: Env, + destination: BackupDestinationRecord, + blobName: string +): Promise { + const id = env.BACKUP_TRANSFER_RUNNER.idFromName('remote-attachment-restore'); + const stub = env.BACKUP_TRANSFER_RUNNER.get(id); + const response = await stub.fetch('https://backup-transfer/internal/download-remote-attachment', { + method: 'POST', + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + body: JSON.stringify({ + destination, + blobName, + }), + }); + if (response.status === 404) { + return null; + } + if (!response.ok) { + throw new Error(`Remote attachment download failed: ${response.status}`); + } + return new Uint8Array(await response.arrayBuffer()); +} + +async function downloadRemoteAttachmentBatchViaDurableObject( + env: Env, + destination: BackupDestinationRecord, + blobNames: string[] +): Promise> { + const names = Array.from(new Set(blobNames.map((blobName) => String(blobName || '').trim()).filter(Boolean))); + const result = new Map(); + if (!names.length) return result; + + const id = env.BACKUP_TRANSFER_RUNNER.idFromName('remote-attachment-restore'); + const stub = env.BACKUP_TRANSFER_RUNNER.get(id); + const response = await stub.fetch('https://backup-transfer/internal/download-remote-attachment-batch', { + method: 'POST', + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + body: JSON.stringify({ + destination, + blobNames: names, + }), + }); + if (!response.ok) { + throw new Error(`Remote attachment batch download failed: ${response.status}`); + } + + const files = unzipSync(new Uint8Array(await response.arrayBuffer())); + const manifestBytes = files['manifest.json']; + if (!manifestBytes) return result; + const manifest = JSON.parse(new TextDecoder().decode(manifestBytes)) as { + entries?: Array<{ blobName?: string; path?: string }>; + }; + for (const entry of manifest.entries || []) { + const blobName = String(entry.blobName || '').trim(); + const path = String(entry.path || '').trim(); + const bytes = path ? files[path] : null; + if (blobName && bytes) { + result.set(blobName, bytes); + } + } + return result; +} + +function collectExternalRemoteAttachmentBlobNames(archiveBytes: Uint8Array): string[] { + const parsed = parseBackupArchive(archiveBytes, { allowExternalAttachmentBlobs: true }); + const refs = new Map( + (parsed.payload.manifest.attachmentBlobs || []) + .map((item) => [`${String(item.cipherId || '').trim()}/${String(item.attachmentId || '').trim()}`, item]) + ); + const names: string[] = []; + const seen = new Set(); + + for (const row of parsed.payload.db.attachments || []) { + const cipherId = String(row.cipher_id || '').trim(); + const attachmentId = String(row.id || '').trim(); + const inlinePath = `attachments/${cipherId}/${attachmentId}.bin`; + if (parsed.files[inlinePath]) continue; + const ref = refs.get(`${cipherId}/${attachmentId}`); + const blobName = String(ref?.blobName || '').trim(); + if (blobName && !seen.has(blobName)) { + seen.add(blobName); + names.push(blobName); + } + } + + return names; +} + function toImportStatusCode(message: string): number { const lower = message.toLowerCase(); + if (lower.includes('checksum')) return 400; if (lower.includes('invalid backup') || lower.includes('invalid json')) return 400; if (lower.includes('fresh instance')) return 409; if (lower.includes('not configured') || lower.includes('kv')) return 409; return 500; } +export async function importAndAuditRemoteBackupFile( + env: Env, + storage: StorageService, + actorUserId: string, + remoteFile: RemoteBackupFile, + destination: BackupDestinationRecord, + remotePath: string, + replaceExisting: boolean, + checksumMismatchAccepted: boolean, + auditMetadata: Record | null = null, + targetDeviceIdentifier: string | null = null +): Promise { + const restoreFileName = remoteFile.fileName || remotePath.split('/').pop() || remotePath; + const externalAttachmentBlobNames = collectExternalRemoteAttachmentBlobNames(remoteFile.bytes); + const externalAttachmentCache = new Map(); + const progress: BackupRestoreProgressReporter = async (event) => { + await notifyUserBackupRestoreProgress( + env, + actorUserId, + { + operation: 'backup-restore', + ...event, + }, + targetDeviceIdentifier + ); + }; + const result = await importRemoteBackupArchiveBytes( + remoteFile.bytes, + env, + actorUserId, + replaceExisting, + { + loadAttachment: async (blobName) => { + const normalized = String(blobName || '').trim(); + if (!normalized) return null; + if (externalAttachmentCache.has(normalized)) { + return externalAttachmentCache.get(normalized) || null; + } + + const start = Math.max(0, externalAttachmentBlobNames.indexOf(normalized)); + const batchNames = externalAttachmentBlobNames + .slice(start, start + REMOTE_ATTACHMENT_RESTORE_BATCH_SIZE) + .filter((name) => !externalAttachmentCache.has(name)); + if (!batchNames.includes(normalized)) { + batchNames.unshift(normalized); + } + + try { + const batch = await downloadRemoteAttachmentBatchViaDurableObject(env, destination, batchNames); + for (const name of batchNames) { + externalAttachmentCache.set(name, batch.get(name) || null); + } + } catch { + externalAttachmentCache.set(normalized, await downloadRemoteAttachmentViaDurableObject(env, destination, normalized).catch(() => null)); + } + return externalAttachmentCache.get(normalized) || null; + }, + }, + progress, + restoreFileName + ); + await writeAuditLog(storage, result.auditActorUserId, 'admin.backup.import', 'backup', null, { + users: result.result.imported.users, + ciphers: result.result.imported.ciphers, + attachments: result.result.imported.attachmentFiles, + skippedAttachments: result.result.skipped.attachments, + skippedReason: result.result.skipped.reason, + replaceExisting, + ...getBackupDestinationSummary(destination), + remotePath, + bytes: remoteFile.bytes.byteLength, + trigger: 'remote', + checksumMismatchAccepted, + ...(auditMetadata || {}), + }); + return result; +} + +async function restoreRemoteBackupInDurableObject( + env: Env, + payload: { + actorUserId: string; + allowChecksumMismatch?: boolean; + auditMetadata?: Record | null; + destinationId?: string | null; + path: string; + replaceExisting?: boolean; + targetDeviceIdentifier?: string | null; + } +): Promise { + const id = env.BACKUP_TRANSFER_RUNNER.idFromName('configured-backup-runner'); + const stub = env.BACKUP_TRANSFER_RUNNER.get(id); + const response = await stub.fetch('https://backup-transfer/internal/restore-remote-backup', { + method: 'POST', + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + body: JSON.stringify(payload), + }); + if (response.status === 409) { + return null; + } + if (!response.ok) { + let message = `Remote backup restore failed: ${response.status}`; + try { + const body = await response.json<{ error?: string }>(); + if (body?.error) message = body.error; + } catch { + // Preserve the status-based message when the DO returns a non-JSON error. + } + throw new Error(message); + } + return response.json(); +} + async function runImportAndAudit( env: Env, request: Request, @@ -788,76 +1001,22 @@ export async function handleRestoreAdminRemoteBackup(request: Request, env: Env, return errorResponse('Remote restore payload is invalid', 400); } - const storage = new StorageService(env.DB); try { - const settings = await loadBackupSettings(storage, env, 'UTC'); - const destination = requireBackupDestination(settings, body.destinationId || null); const path = ensureRemoteRestoreCandidate(String(body.path || '')); const targetDeviceIdentifier = String(request.headers.get('X-NodeWarden-Acting-Device-Id') || '').trim() || null; - const restoreFileNameFromPath = path.split('/').pop() || path; - await notifyUserBackupRestoreProgress( - env, - actorUser.id, - { - operation: 'backup-restore', - source: 'remote', - step: 'remote_fetch_archive', - fileName: restoreFileNameFromPath, - stageTitle: 'txt_backup_restore_progress_remote_fetch_title', - stageDetail: 'txt_backup_restore_progress_remote_fetch_detail', - replaceExisting: !!body.replaceExisting, - }, - targetDeviceIdentifier - ); - const remoteFile = await downloadRemoteBackupFile(destination, path); - const checksumOk = await verifyBackupArchiveFileNameChecksum(remoteFile.bytes, remoteFile.fileName || path); - if (!checksumOk && !body.allowChecksumMismatch) { - return errorResponse('Remote backup file checksum does not match its filename', 400); + const imported = await restoreRemoteBackupInDurableObject(env, { + actorUserId: actorUser.id, + allowChecksumMismatch: !!body.allowChecksumMismatch, + auditMetadata: auditRequestMetadata(request), + destinationId: body.destinationId || null, + path, + replaceExisting: !!body.replaceExisting, + targetDeviceIdentifier, + }); + if (!imported) { + return errorResponse('Another backup or restore run is already in progress', 409); } - const restoreFileName = remoteFile.fileName || path.split('/').pop() || path; - const progress: BackupRestoreProgressReporter = async (event) => { - await notifyUserBackupRestoreProgress( - env, - actorUser.id, - { - operation: 'backup-restore', - ...event, - }, - targetDeviceIdentifier - ); - }; - const imported = await (async () => { - const storage = new StorageService(env.DB); - const result = await importRemoteBackupArchiveBytes( - remoteFile.bytes, - env, - actorUser.id, - !!body.replaceExisting, - { - loadAttachment: async (blobName) => { - const file = await downloadRemoteBackupFile(destination, `attachments/${blobName}`).catch(() => null); - return file?.bytes || null; - }, - }, - progress, - restoreFileName - ); - await writeAuditLog(storage, result.auditActorUserId, 'admin.backup.import', 'backup', null, { - users: result.result.imported.users, - ciphers: result.result.imported.ciphers, - attachments: result.result.imported.attachmentFiles, - skippedAttachments: result.result.skipped.attachments, - skippedReason: result.result.skipped.reason, - replaceExisting: !!body.replaceExisting, - ...getBackupDestinationSummary(destination), - remotePath: path, - bytes: remoteFile.bytes.byteLength, - trigger: 'remote', - checksumMismatchAccepted: !checksumOk, - }, request); - return result; - })(); - return jsonResponse(imported.result); + return jsonResponse(imported); } catch (error) { const message = error instanceof Error ? error.message : 'Remote backup restore failed'; return errorResponse(message, toImportStatusCode(message));