feat: enhance backup and restore functionality with integrity checks and progress tracking

- Added support for backup integrity verification during export and restore processes. - Introduced progress dispatching for backup export and restore operations. - Implemented new API endpoints for inspecting remote backup integrity. - Enhanced user interface with progress indicators and warning dialogs for integrity issues. - Updated localization strings for new features and user feedback. - Refactored backup-related functions for better clarity and maintainability.
2026-06-20 13:00:39 +00:00 · 2026-03-28 05:52:47 +08:00
parent bd8e26d2ab
commit 2a7879efaa
18 changed files with 2250 additions and 225 deletions
@@ -1,7 +1,12 @@
 import type { Env, User } from '../types';
 import { errorResponse, jsonResponse } from '../utils/response';
 import { generateUUID } from '../utils/uuid';
-import { type BackupArchiveBundle, buildBackupArchive } from '../services/backup-archive';
+import {
+  type BackupArchiveBundle,
+  buildBackupArchive,
+  inspectBackupArchiveFileNameChecksum,
+  verifyBackupArchiveFileNameChecksum,
+} from '../services/backup-archive';
 import {
  type BackupDestinationRecord,
  type BackupSettingsInput,
@@ -17,19 +22,25 @@ import {
  requireBackupDestination,
  saveBackupSettings,
 } from '../services/backup-config';
-import { type BackupImportExecutionResult, importBackupArchiveBytes, importRemoteBackupArchiveBytes } from '../services/backup-import';
 import {
+  type BackupImportExecutionResult,
+  type BackupRestoreProgressReporter,
+  importBackupArchiveBytes,
+  importRemoteBackupArchiveBytes,
+} from '../services/backup-import';
+import {
+  type RemoteBackupTransferSession,
+  createRemoteBackupTransferSession,
  deleteRemoteBackupFile,
  downloadRemoteBackupFile,
  ensureRemoteRestoreCandidate,
  listRemoteBackupEntries,
  pruneRemoteBackupArchives,
-  remoteBackupFileExists,
-  uploadRemoteBackupFile,
  uploadBackupArchive,
 } from '../services/backup-uploader';
 import { StorageService } from '../services/storage';
 import { getBlobObject } from '../services/blob-store';
+import { notifyUserBackupProgress, notifyUserBackupRestoreProgress } from '../durable/notifications-hub';

 function isAdmin(user: User): boolean {
  return user.role === 'admin' && user.status === 'active';
@@ -81,13 +92,74 @@ function ensureBackupBlobName(value: string): string {
  return parts.join('/');
 }

+const REMOTE_ATTACHMENT_INDEX_PATH = 'attachments/.nodewarden-attachment-index.v1.json';
+
+interface RemoteAttachmentIndexPayload {
+  version: 1;
+  blobs: Record<string, { sizeBytes: number; updatedAt: string }>;
+}
+
+async function loadRemoteAttachmentIndex(session: RemoteBackupTransferSession): Promise<Map<string, number>> {
+  try {
+    const file = await session.download(REMOTE_ATTACHMENT_INDEX_PATH);
+    const payload = JSON.parse(new TextDecoder().decode(file.bytes)) as RemoteAttachmentIndexPayload;
+    if (payload?.version !== 1 || !payload.blobs || typeof payload.blobs !== 'object') {
+      return new Map<string, number>();
+    }
+    return new Map(
+      Object.entries(payload.blobs)
+        .filter(([key, value]) => !!String(key || '').trim() && Number.isFinite(Number(value?.sizeBytes || 0)))
+        .map(([key, value]) => [key, Number(value.sizeBytes || 0)])
+    );
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (message.includes('404') || message.includes('Please select a backup file')) {
+      return new Map<string, number>();
+    }
+    throw error;
+  }
+}
+
+async function saveRemoteAttachmentIndex(
+  session: RemoteBackupTransferSession,
+  index: Map<string, number>
+): Promise<void> {
+  const payload: RemoteAttachmentIndexPayload = {
+    version: 1,
+    blobs: Object.fromEntries(
+      Array.from(index.entries()).map(([blobName, sizeBytes]) => [
+        blobName,
+        {
+          sizeBytes,
+          updatedAt: new Date().toISOString(),
+        },
+      ])
+    ),
+  };
+  const bytes = new TextEncoder().encode(JSON.stringify(payload));
+  await session.putFile(REMOTE_ATTACHMENT_INDEX_PATH, bytes, {
+    contentType: 'application/json; charset=utf-8',
+  });
+}
+
 async function executeConfiguredBackup(
  env: Env,
  storage: StorageService,
  actorUserId: string | null,
  trigger: 'manual' | 'scheduled',
-  destinationId?: string | null
+  destinationId?: string | null,
+  progress?: ((event: {
+    operation: 'backup-remote-run';
+    step: string;
+    fileName: string;
+    stageTitle: string;
+    stageDetail: string;
+    done?: boolean;
+    ok?: boolean;
+    error?: string | null;
+  }) => Promise<void>) | null
 ): Promise<{ fileName: string; fileSize: number; remotePath: string; provider: string }> {
+  const maxArchiveUploadAttempts = 3;
  const currentSettings = await loadBackupSettings(storage, env, 'UTC');
  const destination = requireBackupDestination(currentSettings, destinationId);

@@ -99,25 +171,109 @@ async function executeConfiguredBackup(
  await saveBackupSettings(storage, env, currentSettings);

  try {
+    await progress?.({
+      operation: 'backup-remote-run',
+      step: 'remote_run_prepare',
+      fileName: '',
+      stageTitle: 'txt_backup_remote_run_progress_prepare_title',
+      stageDetail: 'txt_backup_remote_run_progress_prepare_detail',
+    });
    const archive = await buildBackupArchive(env, now, {
      includeAttachments: destination.includeAttachments,
+      progress: progress
+        ? async (event) => {
+          if (event.step === 'archive_ready') {
+            return;
+          }
+          await progress({
+            operation: 'backup-remote-run',
+            step: `remote_run_${event.step}`,
+            fileName: event.fileName || '',
+            stageTitle: event.stageTitle,
+            stageDetail: event.stageDetail,
+          });
+        }
+        : undefined,
    });
+    await progress?.({
+      operation: 'backup-remote-run',
+      step: 'remote_run_sync_attachments',
+      fileName: archive.fileName,
+      stageTitle: 'txt_backup_remote_run_progress_sync_attachments_title',
+      stageDetail: destination.includeAttachments
+        ? 'txt_backup_remote_run_progress_sync_attachments_detail'
+        : 'txt_backup_remote_run_progress_sync_attachments_skipped_detail',
+    });
+    const remoteSession = createRemoteBackupTransferSession(destination);
+    const remoteAttachmentIndex = await loadRemoteAttachmentIndex(remoteSession);
+    let attachmentIndexChanged = false;
    for (const attachment of archive.manifest.attachmentBlobs || []) {
+      if (remoteAttachmentIndex.get(attachment.blobName) === attachment.sizeBytes) {
+        continue;
+      }
      const remotePath = `attachments/${attachment.blobName}`;
-      if (await remoteBackupFileExists(destination, remotePath)) continue;
      const object = await getBlobObject(env, attachment.blobName);
      if (!object) {
        throw new Error(`Attachment blob missing for ${attachment.blobName}`);
      }
      const bytes = new Uint8Array(await new Response(object.body).arrayBuffer());
-      await uploadRemoteBackupFile(destination, remotePath, bytes, {
+      await remoteSession.putFile(remotePath, bytes, {
        contentType: object.contentType,
      });
+      remoteAttachmentIndex.set(attachment.blobName, attachment.sizeBytes);
+      attachmentIndexChanged = true;
+    }
+    if (attachmentIndexChanged) {
+      await saveRemoteAttachmentIndex(remoteSession, remoteAttachmentIndex);
+    }
+    let upload: Awaited<ReturnType<typeof uploadBackupArchive>> | null = null;
+    for (let attempt = 1; attempt <= maxArchiveUploadAttempts; attempt++) {
+      await progress?.({
+        operation: 'backup-remote-run',
+        step: 'remote_run_upload_archive',
+        fileName: archive.fileName,
+        stageTitle: 'txt_backup_remote_run_progress_upload_title',
+        stageDetail: 'txt_backup_remote_run_progress_upload_detail',
+      });
+      upload = await remoteSession.uploadArchive(archive.bytes, archive.fileName);
+      try {
+        await progress?.({
+          operation: 'backup-remote-run',
+          step: 'remote_run_verify_archive',
+          fileName: archive.fileName,
+          stageTitle: 'txt_backup_remote_run_progress_verify_title',
+          stageDetail: 'txt_backup_remote_run_progress_verify_detail',
+        });
+        const remoteFile = await remoteSession.download(archive.fileName);
+        const checksumOk = await verifyBackupArchiveFileNameChecksum(remoteFile.bytes, archive.fileName);
+        if (!checksumOk) {
+          throw new Error('Remote backup ZIP checksum verification failed');
+        }
+        if (remoteFile.bytes.byteLength !== archive.bytes.byteLength) {
+          throw new Error('Remote backup ZIP size verification failed');
+        }
+        break;
+      } catch (error) {
+        await remoteSession.deleteFile(archive.fileName).catch(() => undefined);
+        if (attempt === maxArchiveUploadAttempts) {
+          const message = error instanceof Error ? error.message : 'Remote backup ZIP verification failed';
+          throw new Error(`Backup archive upload verification failed after ${maxArchiveUploadAttempts} attempts: ${message}`);
+        }
+      }
+    }
+    if (!upload) {
+      throw new Error('Backup archive upload failed');
    }
-    const upload = await uploadBackupArchive(destination, archive.bytes, archive.fileName);
    let prunedFileCount = 0;
    let pruneErrorMessage: string | null = null;
    try {
+      await progress?.({
+        operation: 'backup-remote-run',
+        step: 'remote_run_cleanup',
+        fileName: archive.fileName,
+        stageTitle: 'txt_backup_remote_run_progress_cleanup_title',
+        stageDetail: 'txt_backup_remote_run_progress_cleanup_detail',
+      });
      prunedFileCount = await pruneRemoteBackupArchives(destination, destination.schedule.retentionCount, archive.fileName);
    } catch (error) {
      pruneErrorMessage = error instanceof Error ? error.message : 'Old backup cleanup failed';
@@ -137,10 +293,21 @@ async function executeConfiguredBackup(
      remotePath: upload.remotePath,
      fileName: archive.fileName,
      fileBytes: archive.bytes.byteLength,
+      uploadVerificationAttempts: maxArchiveUploadAttempts,
      prunedFileCount,
      pruneError: pruneErrorMessage,
    });

+    await progress?.({
+      operation: 'backup-remote-run',
+      step: 'remote_run_complete',
+      fileName: archive.fileName,
+      stageTitle: 'txt_backup_remote_run_progress_complete_title',
+      stageDetail: 'txt_backup_remote_run_progress_complete_detail',
+      done: true,
+      ok: true,
+    });
+
    return {
      fileName: archive.fileName,
      fileSize: archive.bytes.byteLength,
@@ -156,6 +323,16 @@ async function executeConfiguredBackup(
      ...getBackupDestinationSummary(destination),
      error: destination.runtime.lastErrorMessage,
    });
+    await progress?.({
+      operation: 'backup-remote-run',
+      step: 'remote_run_failed',
+      fileName: '',
+      stageTitle: 'txt_backup_remote_run_progress_failed_title',
+      stageDetail: 'txt_backup_remote_run_progress_failed_detail',
+      done: true,
+      ok: false,
+      error: destination.runtime.lastErrorMessage,
+    });
    throw error;
  }
 }
@@ -170,13 +347,35 @@ function toImportStatusCode(message: string): number {

 async function runImportAndAudit(
  env: Env,
+  request: Request,
  actorUser: User,
  archiveBytes: Uint8Array,
+  fileName: string,
  replaceExisting: boolean,
  metadata: Record<string, unknown>
 ): Promise<BackupImportExecutionResult> {
  const storage = new StorageService(env.DB);
-  const imported = await importBackupArchiveBytes(archiveBytes, env, actorUser.id, replaceExisting);
+  const targetDeviceIdentifier = String(request.headers.get('X-NodeWarden-Acting-Device-Id') || '').trim() || null;
+  const progress: BackupRestoreProgressReporter = async (event) => {
+    await notifyUserBackupRestoreProgress(
+      env,
+      actorUser.id,
+      {
+        operation: 'backup-restore',
+        ...event,
+      },
+      targetDeviceIdentifier
+    );
+  };
+  await progress({
+    source: 'local',
+    step: 'local_upload_received',
+    fileName,
+    stageTitle: 'txt_backup_restore_progress_local_upload_title',
+    stageDetail: 'txt_backup_restore_progress_local_upload_detail',
+    replaceExisting,
+  });
+  const imported = await importBackupArchiveBytes(archiveBytes, env, actorUser.id, replaceExisting, progress, fileName);
  await writeAuditLog(storage, imported.auditActorUserId, 'admin.backup.import', 'backup', null, {
    users: imported.result.imported.users,
    ciphers: imported.result.imported.ciphers,
@@ -309,7 +508,20 @@ export async function handleRunAdminConfiguredBackup(request: Request, env: Env,
      return errorResponse('Backup run payload is invalid', 400);
    }

-    const result = await executeConfiguredBackup(env, storage, actorUser.id, 'manual', body?.destinationId || null);
+    const targetDeviceIdentifier = String(request.headers.get('X-NodeWarden-Acting-Device-Id') || '').trim() || null;
+    const progress = async (event: {
+      operation: 'backup-remote-run';
+      step: string;
+      fileName: string;
+      stageTitle: string;
+      stageDetail: string;
+      done?: boolean;
+      ok?: boolean;
+      error?: string | null;
+    }) => {
+      await notifyUserBackupProgress(env, actorUser.id, event, targetDeviceIdentifier);
+    };
+    const result = await executeConfiguredBackup(env, storage, actorUser.id, 'manual', body?.destinationId || null, progress);
    const settings = await loadBackupSettings(storage, env, 'UTC');
    return jsonResponse({
      object: 'backup-run',
@@ -369,6 +581,29 @@ export async function handleDownloadAdminRemoteBackup(request: Request, env: Env
  }
 }

+export async function handleInspectAdminRemoteBackup(request: Request, env: Env, actorUser: User): Promise<Response> {
+  if (!isAdmin(actorUser)) return errorResponse('Forbidden', 403);
+
+  const storage = new StorageService(env.DB);
+  try {
+    const settings = await loadBackupSettings(storage, env, 'UTC');
+    const url = new URL(request.url);
+    const path = ensureRemoteRestoreCandidate(url.searchParams.get('path') || '');
+    const destination = requireBackupDestination(settings, url.searchParams.get('destinationId') || null);
+    const remoteFile = await downloadRemoteBackupFile(destination, path);
+    const integrity = await inspectBackupArchiveFileNameChecksum(remoteFile.bytes, remoteFile.fileName || path);
+    return jsonResponse({
+      object: 'backup-remote-integrity',
+      destinationId: destination.id,
+      path,
+      fileName: remoteFile.fileName || path.split('/').pop() || path,
+      integrity,
+    });
+  } catch (error) {
+    return errorResponse(error instanceof Error ? error.message : 'Remote backup integrity inspection failed', 409);
+  }
+}
+
 export async function handleDeleteAdminRemoteBackup(request: Request, env: Env, actorUser: User): Promise<Response> {
  if (!isAdmin(actorUser)) return errorResponse('Forbidden', 403);

@@ -392,7 +627,7 @@ export async function handleDeleteAdminRemoteBackup(request: Request, env: Env,
 export async function handleRestoreAdminRemoteBackup(request: Request, env: Env, actorUser: User): Promise<Response> {
  if (!isAdmin(actorUser)) return errorResponse('Forbidden', 403);

-  let body: { destinationId?: string; path?: string; replaceExisting?: boolean };
+  let body: { destinationId?: string; path?: string; replaceExisting?: boolean; allowChecksumMismatch?: boolean };
  try {
    body = await request.json<{ destinationId?: string; path?: string; replaceExisting?: boolean }>();
  } catch {
@@ -404,7 +639,39 @@ export async function handleRestoreAdminRemoteBackup(request: Request, env: Env,
    const settings = await loadBackupSettings(storage, env, 'UTC');
    const destination = requireBackupDestination(settings, body.destinationId || null);
    const path = ensureRemoteRestoreCandidate(String(body.path || ''));
+    const targetDeviceIdentifier = String(request.headers.get('X-NodeWarden-Acting-Device-Id') || '').trim() || null;
+    const restoreFileNameFromPath = path.split('/').pop() || path;
+    await notifyUserBackupRestoreProgress(
+      env,
+      actorUser.id,
+      {
+        operation: 'backup-restore',
+        source: 'remote',
+        step: 'remote_fetch_archive',
+        fileName: restoreFileNameFromPath,
+        stageTitle: 'txt_backup_restore_progress_remote_fetch_title',
+        stageDetail: 'txt_backup_restore_progress_remote_fetch_detail',
+        replaceExisting: !!body.replaceExisting,
+      },
+      targetDeviceIdentifier
+    );
    const remoteFile = await downloadRemoteBackupFile(destination, path);
+    const checksumOk = await verifyBackupArchiveFileNameChecksum(remoteFile.bytes, remoteFile.fileName || path);
+    if (!checksumOk && !body.allowChecksumMismatch) {
+      return errorResponse('Remote backup file checksum does not match its filename', 400);
+    }
+    const restoreFileName = remoteFile.fileName || path.split('/').pop() || path;
+    const progress: BackupRestoreProgressReporter = async (event) => {
+      await notifyUserBackupRestoreProgress(
+        env,
+        actorUser.id,
+        {
+          operation: 'backup-restore',
+          ...event,
+        },
+        targetDeviceIdentifier
+      );
+    };
    const imported = await (async () => {
      const storage = new StorageService(env.DB);
      const result = await importRemoteBackupArchiveBytes(
@@ -413,12 +680,13 @@ export async function handleRestoreAdminRemoteBackup(request: Request, env: Env,
        actorUser.id,
        !!body.replaceExisting,
        {
-          hasAttachment: async (blobName) => remoteBackupFileExists(destination, `attachments/${blobName}`),
-          loadAttachment: async (blobName) => {
-            const file = await downloadRemoteBackupFile(destination, `attachments/${blobName}`).catch(() => null);
-            return file?.bytes || null;
-          },
-        }
+            loadAttachment: async (blobName) => {
+              const file = await downloadRemoteBackupFile(destination, `attachments/${blobName}`).catch(() => null);
+              return file?.bytes || null;
+            },
+        },
+        progress,
+        restoreFileName
      );
      await writeAuditLog(storage, result.auditActorUserId, 'admin.backup.import', 'backup', null, {
        users: result.result.imported.users,
@@ -431,6 +699,7 @@ export async function handleRestoreAdminRemoteBackup(request: Request, env: Env,
        remotePath: path,
        bytes: remoteFile.bytes.byteLength,
        trigger: 'remote',
+        checksumMismatchAccepted: !checksumOk,
      });
      return result;
    })();
@@ -445,6 +714,7 @@ export async function handleAdminExportBackup(request: Request, env: Env, actorU
  if (!isAdmin(actorUser)) return errorResponse('Forbidden', 403);

  const storage = new StorageService(env.DB);
+  const targetDeviceIdentifier = String(request.headers.get('X-NodeWarden-Acting-Device-Id') || '').trim() || null;
  let body: { includeAttachments?: boolean } | null = null;
  try {
    if ((request.headers.get('Content-Type') || '').includes('application/json')) {
@@ -455,11 +725,49 @@ export async function handleAdminExportBackup(request: Request, env: Env, actorU
  }
  let archive: BackupArchiveBundle;
  try {
+    const progress = async (event: {
+      step: string;
+      fileName?: string;
+      stageTitle: string;
+      stageDetail: string;
+      includeAttachments: boolean;
+    }) => {
+      await notifyUserBackupProgress(
+        env,
+        actorUser.id,
+        {
+          operation: 'backup-export',
+          source: 'local',
+          step: `export_${event.step}`,
+          fileName: event.fileName || '',
+          stageTitle: event.stageTitle,
+          stageDetail: event.stageDetail,
+        },
+        targetDeviceIdentifier
+      );
+    };
    archive = await buildBackupArchive(env, new Date(), {
      includeAttachments: !!body?.includeAttachments,
+      progress,
    });
  } catch (error) {
    const message = error instanceof Error ? error.message : 'Backup export failed';
+    await notifyUserBackupProgress(
+      env,
+      actorUser.id,
+      {
+        operation: 'backup-export',
+        source: 'local',
+        step: 'export_failed',
+        fileName: '',
+        stageTitle: 'txt_backup_export_progress_failed_title',
+        stageDetail: 'txt_backup_export_progress_failed_detail',
+        done: true,
+        ok: false,
+        error: message,
+      },
+      targetDeviceIdentifier
+    );
    return errorResponse(message, message.includes('blob missing') ? 409 : 500);
  }

@@ -520,6 +828,7 @@ export async function handleAdminImportBackup(request: Request, env: Env, actorU
  }

  const replaceExisting = String(formData.get('replaceExisting') || '').trim() === '1';
+  const allowChecksumMismatch = String(formData.get('allowChecksumMismatch') || '').trim() === '1';
  let archiveBytes: Uint8Array;
  try {
    archiveBytes = new Uint8Array(await (file as { arrayBuffer(): Promise<ArrayBuffer> }).arrayBuffer());
@@ -528,9 +837,15 @@ export async function handleAdminImportBackup(request: Request, env: Env, actorU
  }

  try {
-    const imported = await runImportAndAudit(env, actorUser, archiveBytes, replaceExisting, {
+    const fileName = 'name' in file ? String((file as File).name || '') : '';
+    const checksumOk = await verifyBackupArchiveFileNameChecksum(archiveBytes, fileName);
+    if (!checksumOk && !allowChecksumMismatch) {
+      return errorResponse('Backup file checksum does not match its filename', 400);
+    }
+    const imported = await runImportAndAudit(env, request, actorUser, archiveBytes, fileName || 'nodewarden_backup.zip', replaceExisting, {
      trigger: 'local',
      bytes: archiveBytes.byteLength,
+      checksumMismatchAccepted: !checksumOk,
    });
    return jsonResponse(imported.result);
  } catch (error) {