feat: enhance backup import functionality with locking mechanism and checksum support

2026-08-05 06:50:10 +00:00 · 2026-06-26 18:45:23 +08:00
parent 8942e5bd49
commit e9272ec29a
13 changed files with 370 additions and 135 deletions
@@ -26,6 +26,7 @@ import {
 } from '../../shared/backup-schema';

 export const BACKUP_SETTINGS_CONFIG_KEY = 'backup.settings.v1';
+const BACKUP_RUNTIME_CONFIG_KEY = 'backup.runtime.v1';
 export const BACKUP_SCHEDULER_WINDOW_MINUTES = 5;
 const MAX_BACKUP_DESTINATIONS = 24;

@@ -324,6 +325,47 @@ function mapDestinationsById(destinations: BackupDestinationRecord[]): Map<strin
  return new Map(destinations.map((destination) => [destination.id, destination]));
 }

+function stripRuntimeFromSettings(settings: BackupSettings): BackupSettings {
+  return {
+    destinations: settings.destinations.map((destination) => ({
+      ...destination,
+      runtime: normalizeRuntime(null),
+    })),
+  };
+}
+
+function serializeRuntimeState(settings: BackupSettings): string {
+  return JSON.stringify({
+    version: 1,
+    destinations: Object.fromEntries(
+      settings.destinations.map((destination) => [destination.id, normalizeRuntime(destination.runtime)])
+    ),
+  });
+}
+
+async function loadBackupRuntimeStates(storage: StorageService): Promise<Map<string, BackupRuntimeState>> {
+  const raw = await storage.getConfigValue(BACKUP_RUNTIME_CONFIG_KEY);
+  if (!raw) return new Map();
+  try {
+    const parsed = JSON.parse(raw) as { destinations?: Record<string, unknown> };
+    const entries = Object.entries(parsed.destinations || {})
+      .filter(([id]) => !!asTrimmedString(id))
+      .map(([id, runtime]) => [id, normalizeRuntime(runtime)] as const);
+    return new Map(entries);
+  } catch {
+    return new Map();
+  }
+}
+
+function mergeRuntimeStates(settings: BackupSettings, runtimes: Map<string, BackupRuntimeState>): BackupSettings {
+  return {
+    destinations: settings.destinations.map((destination) => ({
+      ...destination,
+      runtime: runtimes.get(destination.id) || normalizeRuntime(destination.runtime),
+    })),
+  };
+}
+
 export function getDefaultBackupSettings(timezone: string = 'UTC'): BackupSettings {
  return createSharedDefaultBackupSettings(assertValidTimeZone(timezone));
 }
@@ -387,27 +429,30 @@ export function normalizeBackupSettingsInput(
 }

 export function serializeBackupSettings(settings: BackupSettings): string {
-  return JSON.stringify(settings);
+  return JSON.stringify(stripRuntimeFromSettings(settings));
 }

 export async function loadBackupSettings(storage: StorageService, env: Env, fallbackTimezone: string = 'UTC'): Promise<BackupSettings> {
  const raw = await storage.getConfigValue(BACKUP_SETTINGS_CONFIG_KEY);
+  const mergeRuntime = async (settings: BackupSettings): Promise<BackupSettings> => (
+    mergeRuntimeStates(settings, await loadBackupRuntimeStates(storage))
+  );
  if (!raw) {
    const settings = getDefaultBackupSettings(fallbackTimezone);
    await saveBackupSettings(storage, env, settings);
-    return settings;
+    return mergeRuntime(settings);
  }

  const envelope = parseBackupSettingsEnvelope(raw);
  if (!envelope) {
    const settings = parseBackupSettings(raw, fallbackTimezone);
    await saveBackupSettings(storage, env, settings);
-    return settings;
+    return mergeRuntime(settings);
  }

  try {
    const decrypted = await decryptBackupSettingsRuntime(raw, env);
-    return parseBackupSettings(decrypted, fallbackTimezone);
+    return mergeRuntime(parseBackupSettings(decrypted, fallbackTimezone));
  } catch {
    throw new Error('Backup settings need administrator reactivation after restore');
  }
@@ -417,6 +462,27 @@ export async function saveBackupSettings(storage: StorageService, env: Env, sett
  const users = await storage.getAllUsers();
  const encrypted = await encryptBackupSettingsEnvelope(serializeBackupSettings(settings), env, users);
  await storage.setConfigValue(BACKUP_SETTINGS_CONFIG_KEY, encrypted);
+  await saveBackupRuntimeStates(storage, settings);
+}
+
+export async function saveBackupRuntimeStates(storage: StorageService, settings: BackupSettings): Promise<void> {
+  await storage.setConfigValue(BACKUP_RUNTIME_CONFIG_KEY, serializeRuntimeState(settings));
+}
+
+export async function updateBackupDestinationRuntime(
+  storage: StorageService,
+  destinationId: string,
+  mutator: (runtime: BackupRuntimeState) => BackupRuntimeState
+): Promise<BackupRuntimeState> {
+  const runtimes = await loadBackupRuntimeStates(storage);
+  const current = runtimes.get(destinationId) || normalizeRuntime(null);
+  const next = normalizeRuntime(mutator(current));
+  runtimes.set(destinationId, next);
+  await storage.setConfigValue(BACKUP_RUNTIME_CONFIG_KEY, JSON.stringify({
+    version: 1,
+    destinations: Object.fromEntries(runtimes.entries()),
+  }));
+  return next;
 }

 export async function normalizeImportedBackupSettings(storage: StorageService, env: Env, fallbackTimezone: string = 'UTC'): Promise<void> {
@@ -596,9 +662,9 @@ export function hasBackupSlotBetween(
  const endMs = endExclusive.getTime();
  if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs <= startMs) return false;

-  const lastAttemptAt = destination.runtime.lastAttemptAt ? new Date(destination.runtime.lastAttemptAt) : null;
-  const lastAttemptMs = lastAttemptAt && Number.isFinite(lastAttemptAt.getTime())
-    ? lastAttemptAt.getTime()
+  const lastSuccessAt = destination.runtime.lastSuccessAt ? new Date(destination.runtime.lastSuccessAt) : null;
+  const lastSuccessMs = lastSuccessAt && Number.isFinite(lastSuccessAt.getTime())
+    ? lastSuccessAt.getTime()
    : Number.NEGATIVE_INFINITY;

  const dayCursor = new Date(startMs);
@@ -620,7 +686,7 @@ export function hasBackupSlotBetween(
      for (const slotStart of slotStarts) {
        const slotStartMs = slotStart.getTime();
        if (slotStartMs < startMs || slotStartMs >= endMs) continue;
-        if (lastAttemptMs >= slotStartMs) continue;
+        if (lastSuccessMs >= slotStartMs) continue;
        return true;
      }
    }
@@ -637,9 +703,9 @@ export function isBackupDueNow(
 ): boolean {
  if (!destination.schedule.enabled) return false;
  const toleranceMs = Math.max(1, windowMinutes) * 60 * 1000;
-  const lastAttemptAt = destination.runtime.lastAttemptAt ? new Date(destination.runtime.lastAttemptAt) : null;
-  const lastAttemptMs = lastAttemptAt && Number.isFinite(lastAttemptAt.getTime())
-    ? lastAttemptAt.getTime()
+  const lastSuccessAt = destination.runtime.lastSuccessAt ? new Date(destination.runtime.lastSuccessAt) : null;
+  const lastSuccessMs = lastSuccessAt && Number.isFinite(lastSuccessAt.getTime())
+    ? lastSuccessAt.getTime()
    : Number.NEGATIVE_INFINITY;
  const localDateKey = getBackupLocalDateKey(now, destination.schedule.timezone);
  const slotStarts = getBackupSlotStartsForLocalDay(
@@ -652,7 +718,7 @@ export function isBackupDueNow(
  for (const slotStart of slotStarts) {
    const slotStartMs = slotStart.getTime();
    if (now.getTime() < slotStartMs || now.getTime() >= slotStartMs + toleranceMs) continue;
-    if (lastAttemptMs >= slotStartMs) return false;
+    if (lastSuccessMs >= slotStartMs) return false;
    return true;
  }
  return false;
@@ -33,6 +33,13 @@ export interface RemoteBackupFile {
  bytes: Uint8Array;
 }

+export interface RemoteBackupFileStat {
+  provider: BackupDestinationType;
+  remotePath: string;
+  size: number | null;
+  modifiedAt: string | null;
+}
+
 export interface RemoteBackupFilePutOptions {
  contentType?: string;
 }
@@ -433,6 +440,10 @@ async function deleteFromWebDav(config: WebDavBackupDestination, relativePath: s
 }

 async function existsInWebDav(config: WebDavBackupDestination, relativePath: string): Promise<boolean> {
+  return (await statWebDavFile(config, relativePath)) !== null;
+}
+
+async function statWebDavFile(config: WebDavBackupDestination, relativePath: string): Promise<RemoteBackupFileStat | null> {
  const authHeader = toBasicAuthHeader(config.username, config.password);
  const remotePath = webDavFullPath(config, relativePath);
  const response = await fetch(buildWebDavUrl(config.baseUrl, remotePath), {
@@ -441,11 +452,17 @@ async function existsInWebDav(config: WebDavBackupDestination, relativePath: str
      Authorization: authHeader,
    },
  });
-  if (response.status === 404) return false;
+  if (response.status === 404) return null;
  if (!response.ok) {
    throw new Error(`WebDAV existence check failed: ${response.status}`);
  }
-  return true;
+  const size = Number(response.headers.get('Content-Length') || '');
+  return {
+    provider: 'webdav',
+    remotePath: normalizeRelativePath(relativePath),
+    size: Number.isFinite(size) ? size : null,
+    modifiedAt: parseHttpDate(response.headers.get('Last-Modified') || ''),
+  };
 }

 function isBucketHostedS3Endpoint(endpoint: URL, bucket: string): boolean {
@@ -540,61 +557,68 @@ async function listS3Entries(config: S3BackupDestination, relativePath: string):
  const currentPath = normalizeRelativePath(relativePath);
  const targetPrefixBase = normalizeS3ObjectKey(config, currentPath);
  const targetPrefix = trimSlashes(targetPrefixBase) ? `${trimSlashes(targetPrefixBase)}/` : '';
-  const url = s3BucketBaseUrl(config);
-  url.searchParams.set('list-type', '2');
-  url.searchParams.set('delimiter', '/');
-  if (targetPrefix) url.searchParams.set('prefix', targetPrefix);
-
-  const response = await signedS3Request(config, 'GET', url);
-  if (!response.ok) {
-    throw new Error(`S3 listing failed: ${response.status}`);
-  }
-
-  const xml = await response.text();
  const rootPrefix = trimSlashes(config.rootPath);
  const items: RemoteBackupItem[] = [];
+  let continuationToken = '';

-  for (const prefix of extractXmlBlocks(xml, 'CommonPrefixes')) {
-    const fullPrefix = trimSlashes(extractXmlFirst(prefix, 'Prefix') || '');
-    if (!fullPrefix) continue;
-    const relative = rootPrefix
-      ? fullPrefix === rootPrefix
-        ? ''
-        : fullPrefix.startsWith(`${rootPrefix}/`)
-          ? fullPrefix.slice(rootPrefix.length + 1)
+  do {
+    const url = s3BucketBaseUrl(config);
+    url.searchParams.set('list-type', '2');
+    url.searchParams.set('delimiter', '/');
+    if (targetPrefix) url.searchParams.set('prefix', targetPrefix);
+    if (continuationToken) url.searchParams.set('continuation-token', continuationToken);
+
+    const response = await signedS3Request(config, 'GET', url);
+    if (!response.ok) {
+      throw new Error(`S3 listing failed: ${response.status}`);
+    }
+
+    const xml = await response.text();
+
+    for (const prefix of extractXmlBlocks(xml, 'CommonPrefixes')) {
+      const fullPrefix = trimSlashes(extractXmlFirst(prefix, 'Prefix') || '');
+      if (!fullPrefix) continue;
+      const relative = rootPrefix
+        ? fullPrefix === rootPrefix
+          ? ''
+          : fullPrefix.startsWith(`${rootPrefix}/`)
+            ? fullPrefix.slice(rootPrefix.length + 1)
+            : ''
+        : fullPrefix;
+      const normalizedRelative = trimSlashes(relative);
+      if (!normalizedRelative) continue;
+      const itemPath = normalizedRelative.replace(/\/+$/, '');
+      if ((parentPath(itemPath) || '') !== currentPath) continue;
+      items.push({
+        path: itemPath,
+        name: basename(itemPath) || itemPath,
+        isDirectory: true,
+        size: null,
+        modifiedAt: null,
+      });
+    }
+
+    for (const content of extractXmlBlocks(xml, 'Contents')) {
+      const fullKey = trimSlashes(extractXmlFirst(content, 'Key') || '');
+      if (!fullKey || (targetPrefix && fullKey === trimSlashes(targetPrefix))) continue;
+      const relative = rootPrefix
+        ? fullKey.startsWith(`${rootPrefix}/`)
+          ? fullKey.slice(rootPrefix.length + 1)
          : ''
-      : fullPrefix;
-    const normalizedRelative = trimSlashes(relative);
-    if (!normalizedRelative) continue;
-    const itemPath = normalizedRelative.replace(/\/+$/, '');
-    if ((parentPath(itemPath) || '') !== currentPath) continue;
-    items.push({
-      path: itemPath,
-      name: basename(itemPath) || itemPath,
-      isDirectory: true,
-      size: null,
-      modifiedAt: null,
-    });
-  }
+        : fullKey;
+      const normalizedRelative = trimSlashes(relative);
+      if (!normalizedRelative || (parentPath(normalizedRelative) || '') !== currentPath) continue;
+      items.push({
+        path: normalizedRelative,
+        name: basename(normalizedRelative) || normalizedRelative,
+        isDirectory: false,
+        size: Number(extractXmlFirst(content, 'Size') || 0) || null,
+        modifiedAt: parseHttpDate(extractXmlFirst(content, 'LastModified') || '') || null,
+      });
+    }

-  for (const content of extractXmlBlocks(xml, 'Contents')) {
-    const fullKey = trimSlashes(extractXmlFirst(content, 'Key') || '');
-    if (!fullKey || (targetPrefix && fullKey === trimSlashes(targetPrefix))) continue;
-    const relative = rootPrefix
-      ? fullKey.startsWith(`${rootPrefix}/`)
-        ? fullKey.slice(rootPrefix.length + 1)
-        : ''
-      : fullKey;
-    const normalizedRelative = trimSlashes(relative);
-    if (!normalizedRelative || (parentPath(normalizedRelative) || '') !== currentPath) continue;
-    items.push({
-      path: normalizedRelative,
-      name: basename(normalizedRelative) || normalizedRelative,
-      isDirectory: false,
-      size: Number(extractXmlFirst(content, 'Size') || 0) || null,
-      modifiedAt: parseHttpDate(extractXmlFirst(content, 'LastModified') || '') || null,
-    });
-  }
+    continuationToken = extractXmlFirst(xml, 'NextContinuationToken') || '';
+  } while (continuationToken);

  const deduped = new Map<string, RemoteBackupItem>();
  for (const item of items) deduped.set(`${item.isDirectory ? 'd' : 'f'}:${item.path}`, item);
@@ -637,14 +661,24 @@ async function deleteFromS3(config: S3BackupDestination, relativePath: string):
 }

 async function existsInS3(config: S3BackupDestination, relativePath: string): Promise<boolean> {
+  return (await statS3File(config, relativePath)) !== null;
+}
+
+async function statS3File(config: S3BackupDestination, relativePath: string): Promise<RemoteBackupFileStat | null> {
  const objectKey = normalizeS3ObjectKey(config, relativePath);
  const url = s3ObjectUrl(config, objectKey);
  const response = await signedS3Request(config, 'HEAD', url);
-  if (response.status === 404) return false;
+  if (response.status === 404) return null;
  if (!response.ok) {
    throw new Error(`S3 existence check failed: ${response.status}`);
  }
-  return true;
+  const size = Number(response.headers.get('Content-Length') || '');
+  return {
+    provider: 's3',
+    remotePath: normalizeRelativePath(relativePath),
+    size: Number.isFinite(size) ? size : null,
+    modifiedAt: parseHttpDate(response.headers.get('Last-Modified') || ''),
+  };
 }

 interface ConfiguredDestinationAdapter {
@@ -656,6 +690,7 @@ interface ConfiguredDestinationAdapter {
  download: (config: WebDavBackupDestination | S3BackupDestination, relativePath: string) => Promise<RemoteBackupFile>;
  deleteFile: (config: WebDavBackupDestination | S3BackupDestination, relativePath: string) => Promise<void>;
  exists: (config: WebDavBackupDestination | S3BackupDestination, relativePath: string) => Promise<boolean>;
+  stat: (config: WebDavBackupDestination | S3BackupDestination, relativePath: string) => Promise<RemoteBackupFileStat | null>;
 }

 export interface RemoteBackupTransferSession {
@@ -666,6 +701,7 @@ export interface RemoteBackupTransferSession {
  download(relativePath: string): Promise<RemoteBackupFile>;
  deleteFile(relativePath: string): Promise<void>;
  exists(relativePath: string): Promise<boolean>;
+  stat(relativePath: string): Promise<RemoteBackupFileStat | null>;
 }

 function resolveConfiguredDestinationAdapter(
@@ -683,6 +719,7 @@ function resolveConfiguredDestinationAdapter(
      download: (config, relativePath) => downloadFromWebDav(config as WebDavBackupDestination, relativePath),
      deleteFile: (config, relativePath) => deleteFromWebDav(config as WebDavBackupDestination, relativePath),
      exists: (config, relativePath) => existsInWebDav(config as WebDavBackupDestination, relativePath),
+      stat: (config, relativePath) => statWebDavFile(config as WebDavBackupDestination, relativePath),
    };
  }
  if (destination.type === 's3') {
@@ -695,6 +732,7 @@ function resolveConfiguredDestinationAdapter(
      download: (config, relativePath) => downloadFromS3(config as S3BackupDestination, relativePath),
      deleteFile: (config, relativePath) => deleteFromS3(config as S3BackupDestination, relativePath),
      exists: (config, relativePath) => existsInS3(config as S3BackupDestination, relativePath),
+      stat: (config, relativePath) => statS3File(config as S3BackupDestination, relativePath),
    };
  }

@@ -730,6 +768,7 @@ export function createRemoteBackupTransferSession(destination: BackupDestination
    download: async (relativePath: string) => adapter.download(adapter.config, relativePath),
    deleteFile: async (relativePath: string) => adapter.deleteFile(adapter.config, normalizeRelativePath(relativePath)),
    exists: async (relativePath: string) => adapter.exists(adapter.config, normalizeRelativePath(relativePath)),
+    stat: async (relativePath: string) => adapter.stat(adapter.config, normalizeRelativePath(relativePath)),
  };
 }