mirror of
https://github.com/shuaiplus/nodewarden.git
synced 2026-08-05 06:50:10 +00:00
feat: enhance backup import functionality with locking mechanism and checksum support
This commit is contained in:
@@ -209,6 +209,7 @@ export class BackupTransferRunner {
|
||||
}
|
||||
|
||||
let completed = 0;
|
||||
const failures: Array<{ destinationId: string; error: string }> = [];
|
||||
try {
|
||||
await this.touchJob(token);
|
||||
const storage = new StorageService(this.env.DB);
|
||||
@@ -230,21 +231,30 @@ export class BackupTransferRunner {
|
||||
scanStartMs = now.getTime();
|
||||
for (const destination of dueDestinations) {
|
||||
await this.touchJob(token);
|
||||
await executeConfiguredBackup(
|
||||
this.env,
|
||||
storage,
|
||||
null,
|
||||
'scheduled',
|
||||
destination.id,
|
||||
() => this.touchJob(token)
|
||||
);
|
||||
completed += 1;
|
||||
try {
|
||||
await executeConfiguredBackup(
|
||||
this.env,
|
||||
storage,
|
||||
null,
|
||||
'scheduled',
|
||||
destination.id,
|
||||
() => this.touchJob(token)
|
||||
);
|
||||
completed += 1;
|
||||
} catch (error) {
|
||||
failures.push({
|
||||
destinationId: destination.id,
|
||||
error: error instanceof Error ? error.message : 'Scheduled backup failed',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new Response(JSON.stringify({
|
||||
ok: true,
|
||||
completed,
|
||||
failed: failures.length,
|
||||
failures,
|
||||
}), {
|
||||
status: 200,
|
||||
headers: {
|
||||
@@ -318,7 +328,8 @@ export class BackupTransferRunner {
|
||||
replaceExisting,
|
||||
!checksumOk,
|
||||
body.auditMetadata || null,
|
||||
targetDeviceIdentifier
|
||||
targetDeviceIdentifier,
|
||||
() => this.touchJob(token)
|
||||
);
|
||||
|
||||
return new Response(JSON.stringify(result.result), {
|
||||
|
||||
+61
-26
@@ -21,6 +21,7 @@ import {
|
||||
repairBackupSettings,
|
||||
requireBackupDestination,
|
||||
saveBackupSettings,
|
||||
updateBackupDestinationRuntime,
|
||||
} from '../services/backup-config';
|
||||
import {
|
||||
type BackupImportExecutionResult,
|
||||
@@ -260,6 +261,30 @@ async function uploadRemoteAttachmentChunk(
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyUploadedBackupArchive(
|
||||
session: RemoteBackupTransferSession,
|
||||
archive: BackupArchiveBundle
|
||||
): Promise<'metadata' | 'download'> {
|
||||
try {
|
||||
const stat = await session.stat(archive.fileName);
|
||||
if (stat?.size === archive.bytes.byteLength) {
|
||||
return 'metadata';
|
||||
}
|
||||
} catch {
|
||||
// Fall through to a full read-back verification when lightweight metadata is unavailable.
|
||||
}
|
||||
|
||||
const remoteFile = await session.download(archive.fileName);
|
||||
const checksumOk = await verifyBackupArchiveFileNameChecksum(remoteFile.bytes, archive.fileName);
|
||||
if (!checksumOk) {
|
||||
throw new Error('Remote backup ZIP checksum verification failed');
|
||||
}
|
||||
if (remoteFile.bytes.byteLength !== archive.bytes.byteLength) {
|
||||
throw new Error('Remote backup ZIP size verification failed');
|
||||
}
|
||||
return 'download';
|
||||
}
|
||||
|
||||
export async function executeConfiguredBackup(
|
||||
env: Env,
|
||||
storage: StorageService,
|
||||
@@ -287,12 +312,14 @@ export async function executeConfiguredBackup(
|
||||
const destination = requireBackupDestination(currentSettings, destinationId);
|
||||
|
||||
const now = new Date();
|
||||
destination.runtime.lastAttemptAt = now.toISOString();
|
||||
destination.runtime.lastAttemptLocalDate = getBackupLocalDateKey(now, destination.schedule.timezone);
|
||||
destination.runtime.lastErrorAt = null;
|
||||
destination.runtime.lastErrorMessage = null;
|
||||
await touchLease();
|
||||
await saveBackupSettings(storage, env, currentSettings);
|
||||
destination.runtime = await updateBackupDestinationRuntime(storage, destination.id, (runtime) => ({
|
||||
...runtime,
|
||||
lastAttemptAt: now.toISOString(),
|
||||
lastAttemptLocalDate: getBackupLocalDateKey(now, destination.schedule.timezone),
|
||||
lastErrorAt: null,
|
||||
lastErrorMessage: null,
|
||||
}));
|
||||
|
||||
try {
|
||||
await touchLease();
|
||||
@@ -354,6 +381,7 @@ export async function executeConfiguredBackup(
|
||||
}
|
||||
}
|
||||
let upload: Awaited<ReturnType<typeof uploadBackupArchive>> | null = null;
|
||||
let uploadVerificationMethod: 'metadata' | 'download' | null = null;
|
||||
for (let attempt = 1; attempt <= maxArchiveUploadAttempts; attempt++) {
|
||||
await touchLease();
|
||||
await progress?.({
|
||||
@@ -373,14 +401,7 @@ export async function executeConfiguredBackup(
|
||||
stageTitle: 'txt_backup_remote_run_progress_verify_title',
|
||||
stageDetail: 'txt_backup_remote_run_progress_verify_detail',
|
||||
});
|
||||
const remoteFile = await remoteSession.download(archive.fileName);
|
||||
const checksumOk = await verifyBackupArchiveFileNameChecksum(remoteFile.bytes, archive.fileName);
|
||||
if (!checksumOk) {
|
||||
throw new Error('Remote backup ZIP checksum verification failed');
|
||||
}
|
||||
if (remoteFile.bytes.byteLength !== archive.bytes.byteLength) {
|
||||
throw new Error('Remote backup ZIP size verification failed');
|
||||
}
|
||||
uploadVerificationMethod = await verifyUploadedBackupArchive(remoteSession, archive);
|
||||
break;
|
||||
} catch (error) {
|
||||
await remoteSession.deleteFile(archive.fileName).catch(() => undefined);
|
||||
@@ -409,14 +430,16 @@ export async function executeConfiguredBackup(
|
||||
pruneErrorMessage = error instanceof Error ? error.message : 'Old backup cleanup failed';
|
||||
}
|
||||
|
||||
destination.runtime.lastSuccessAt = new Date().toISOString();
|
||||
destination.runtime.lastErrorAt = null;
|
||||
destination.runtime.lastErrorMessage = null;
|
||||
destination.runtime.lastUploadedFileName = archive.fileName;
|
||||
destination.runtime.lastUploadedSizeBytes = archive.bytes.byteLength;
|
||||
destination.runtime.lastUploadedDestination = upload.remotePath;
|
||||
await touchLease();
|
||||
await saveBackupSettings(storage, env, currentSettings);
|
||||
destination.runtime = await updateBackupDestinationRuntime(storage, destination.id, (runtime) => ({
|
||||
...runtime,
|
||||
lastSuccessAt: new Date().toISOString(),
|
||||
lastErrorAt: null,
|
||||
lastErrorMessage: null,
|
||||
lastUploadedFileName: archive.fileName,
|
||||
lastUploadedSizeBytes: archive.bytes.byteLength,
|
||||
lastUploadedDestination: upload.remotePath,
|
||||
}));
|
||||
|
||||
await touchLease();
|
||||
await writeAuditLog(storage, actorUserId, `admin.backup.remote.${trigger}`, 'backup', null, {
|
||||
@@ -426,6 +449,7 @@ export async function executeConfiguredBackup(
|
||||
fileName: archive.fileName,
|
||||
fileBytes: archive.bytes.byteLength,
|
||||
uploadVerificationAttempts: maxArchiveUploadAttempts,
|
||||
uploadVerificationMethod,
|
||||
prunedFileCount,
|
||||
pruneError: pruneErrorMessage,
|
||||
...(auditMetadata || {}),
|
||||
@@ -448,15 +472,18 @@ export async function executeConfiguredBackup(
|
||||
provider: upload.provider,
|
||||
};
|
||||
} catch (error) {
|
||||
destination.runtime.lastErrorAt = new Date().toISOString();
|
||||
destination.runtime.lastErrorMessage = error instanceof Error ? error.message : 'Backup upload failed';
|
||||
const errorMessage = error instanceof Error ? error.message : 'Backup upload failed';
|
||||
await touchLease();
|
||||
await saveBackupSettings(storage, env, currentSettings);
|
||||
destination.runtime = await updateBackupDestinationRuntime(storage, destination.id, (runtime) => ({
|
||||
...runtime,
|
||||
lastErrorAt: new Date().toISOString(),
|
||||
lastErrorMessage: errorMessage,
|
||||
}));
|
||||
|
||||
await touchLease();
|
||||
await writeAuditLog(storage, actorUserId, `admin.backup.remote.${trigger}.failed`, 'backup', null, {
|
||||
...getBackupDestinationSummary(destination),
|
||||
error: destination.runtime.lastErrorMessage,
|
||||
error: errorMessage,
|
||||
...(auditMetadata || {}),
|
||||
});
|
||||
await progress?.({
|
||||
@@ -467,7 +494,7 @@ export async function executeConfiguredBackup(
|
||||
stageDetail: 'txt_backup_remote_run_progress_failed_detail',
|
||||
done: true,
|
||||
ok: false,
|
||||
error: destination.runtime.lastErrorMessage,
|
||||
error: errorMessage,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
@@ -655,12 +682,18 @@ export async function importAndAuditRemoteBackupFile(
|
||||
replaceExisting: boolean,
|
||||
checksumMismatchAccepted: boolean,
|
||||
auditMetadata: Record<string, unknown> | null = null,
|
||||
targetDeviceIdentifier: string | null = null
|
||||
targetDeviceIdentifier: string | null = null,
|
||||
keepAlive?: (() => Promise<void>) | null
|
||||
): Promise<BackupImportExecutionResult> {
|
||||
const touchLease = async () => {
|
||||
await keepAlive?.();
|
||||
};
|
||||
const restoreFileName = remoteFile.fileName || remotePath.split('/').pop() || remotePath;
|
||||
await touchLease();
|
||||
const externalAttachmentBlobNames = collectExternalRemoteAttachmentBlobNames(remoteFile.bytes);
|
||||
const externalAttachmentCache = new Map<string, Uint8Array | null>();
|
||||
const progress: BackupRestoreProgressReporter = async (event) => {
|
||||
await touchLease();
|
||||
await notifyUserBackupRestoreProgress(
|
||||
env,
|
||||
actorUserId,
|
||||
@@ -678,6 +711,7 @@ export async function importAndAuditRemoteBackupFile(
|
||||
replaceExisting,
|
||||
{
|
||||
loadAttachment: async (blobName) => {
|
||||
await touchLease();
|
||||
const normalized = String(blobName || '').trim();
|
||||
if (!normalized) return null;
|
||||
if (externalAttachmentCache.has(normalized)) {
|
||||
@@ -700,6 +734,7 @@ export async function importAndAuditRemoteBackupFile(
|
||||
} catch {
|
||||
externalAttachmentCache.set(normalized, await downloadRemoteAttachmentViaDurableObject(env, destination, normalized).catch(() => null));
|
||||
}
|
||||
await touchLease();
|
||||
return externalAttachmentCache.get(normalized) || null;
|
||||
},
|
||||
},
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
} from '../../shared/backup-schema';
|
||||
|
||||
export const BACKUP_SETTINGS_CONFIG_KEY = 'backup.settings.v1';
|
||||
const BACKUP_RUNTIME_CONFIG_KEY = 'backup.runtime.v1';
|
||||
export const BACKUP_SCHEDULER_WINDOW_MINUTES = 5;
|
||||
const MAX_BACKUP_DESTINATIONS = 24;
|
||||
|
||||
@@ -324,6 +325,47 @@ function mapDestinationsById(destinations: BackupDestinationRecord[]): Map<strin
|
||||
return new Map(destinations.map((destination) => [destination.id, destination]));
|
||||
}
|
||||
|
||||
function stripRuntimeFromSettings(settings: BackupSettings): BackupSettings {
|
||||
return {
|
||||
destinations: settings.destinations.map((destination) => ({
|
||||
...destination,
|
||||
runtime: normalizeRuntime(null),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function serializeRuntimeState(settings: BackupSettings): string {
|
||||
return JSON.stringify({
|
||||
version: 1,
|
||||
destinations: Object.fromEntries(
|
||||
settings.destinations.map((destination) => [destination.id, normalizeRuntime(destination.runtime)])
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
async function loadBackupRuntimeStates(storage: StorageService): Promise<Map<string, BackupRuntimeState>> {
|
||||
const raw = await storage.getConfigValue(BACKUP_RUNTIME_CONFIG_KEY);
|
||||
if (!raw) return new Map();
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as { destinations?: Record<string, unknown> };
|
||||
const entries = Object.entries(parsed.destinations || {})
|
||||
.filter(([id]) => !!asTrimmedString(id))
|
||||
.map(([id, runtime]) => [id, normalizeRuntime(runtime)] as const);
|
||||
return new Map(entries);
|
||||
} catch {
|
||||
return new Map();
|
||||
}
|
||||
}
|
||||
|
||||
function mergeRuntimeStates(settings: BackupSettings, runtimes: Map<string, BackupRuntimeState>): BackupSettings {
|
||||
return {
|
||||
destinations: settings.destinations.map((destination) => ({
|
||||
...destination,
|
||||
runtime: runtimes.get(destination.id) || normalizeRuntime(destination.runtime),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export function getDefaultBackupSettings(timezone: string = 'UTC'): BackupSettings {
|
||||
return createSharedDefaultBackupSettings(assertValidTimeZone(timezone));
|
||||
}
|
||||
@@ -387,27 +429,30 @@ export function normalizeBackupSettingsInput(
|
||||
}
|
||||
|
||||
export function serializeBackupSettings(settings: BackupSettings): string {
|
||||
return JSON.stringify(settings);
|
||||
return JSON.stringify(stripRuntimeFromSettings(settings));
|
||||
}
|
||||
|
||||
export async function loadBackupSettings(storage: StorageService, env: Env, fallbackTimezone: string = 'UTC'): Promise<BackupSettings> {
|
||||
const raw = await storage.getConfigValue(BACKUP_SETTINGS_CONFIG_KEY);
|
||||
const mergeRuntime = async (settings: BackupSettings): Promise<BackupSettings> => (
|
||||
mergeRuntimeStates(settings, await loadBackupRuntimeStates(storage))
|
||||
);
|
||||
if (!raw) {
|
||||
const settings = getDefaultBackupSettings(fallbackTimezone);
|
||||
await saveBackupSettings(storage, env, settings);
|
||||
return settings;
|
||||
return mergeRuntime(settings);
|
||||
}
|
||||
|
||||
const envelope = parseBackupSettingsEnvelope(raw);
|
||||
if (!envelope) {
|
||||
const settings = parseBackupSettings(raw, fallbackTimezone);
|
||||
await saveBackupSettings(storage, env, settings);
|
||||
return settings;
|
||||
return mergeRuntime(settings);
|
||||
}
|
||||
|
||||
try {
|
||||
const decrypted = await decryptBackupSettingsRuntime(raw, env);
|
||||
return parseBackupSettings(decrypted, fallbackTimezone);
|
||||
return mergeRuntime(parseBackupSettings(decrypted, fallbackTimezone));
|
||||
} catch {
|
||||
throw new Error('Backup settings need administrator reactivation after restore');
|
||||
}
|
||||
@@ -417,6 +462,27 @@ export async function saveBackupSettings(storage: StorageService, env: Env, sett
|
||||
const users = await storage.getAllUsers();
|
||||
const encrypted = await encryptBackupSettingsEnvelope(serializeBackupSettings(settings), env, users);
|
||||
await storage.setConfigValue(BACKUP_SETTINGS_CONFIG_KEY, encrypted);
|
||||
await saveBackupRuntimeStates(storage, settings);
|
||||
}
|
||||
|
||||
export async function saveBackupRuntimeStates(storage: StorageService, settings: BackupSettings): Promise<void> {
|
||||
await storage.setConfigValue(BACKUP_RUNTIME_CONFIG_KEY, serializeRuntimeState(settings));
|
||||
}
|
||||
|
||||
export async function updateBackupDestinationRuntime(
|
||||
storage: StorageService,
|
||||
destinationId: string,
|
||||
mutator: (runtime: BackupRuntimeState) => BackupRuntimeState
|
||||
): Promise<BackupRuntimeState> {
|
||||
const runtimes = await loadBackupRuntimeStates(storage);
|
||||
const current = runtimes.get(destinationId) || normalizeRuntime(null);
|
||||
const next = normalizeRuntime(mutator(current));
|
||||
runtimes.set(destinationId, next);
|
||||
await storage.setConfigValue(BACKUP_RUNTIME_CONFIG_KEY, JSON.stringify({
|
||||
version: 1,
|
||||
destinations: Object.fromEntries(runtimes.entries()),
|
||||
}));
|
||||
return next;
|
||||
}
|
||||
|
||||
export async function normalizeImportedBackupSettings(storage: StorageService, env: Env, fallbackTimezone: string = 'UTC'): Promise<void> {
|
||||
@@ -596,9 +662,9 @@ export function hasBackupSlotBetween(
|
||||
const endMs = endExclusive.getTime();
|
||||
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs <= startMs) return false;
|
||||
|
||||
const lastAttemptAt = destination.runtime.lastAttemptAt ? new Date(destination.runtime.lastAttemptAt) : null;
|
||||
const lastAttemptMs = lastAttemptAt && Number.isFinite(lastAttemptAt.getTime())
|
||||
? lastAttemptAt.getTime()
|
||||
const lastSuccessAt = destination.runtime.lastSuccessAt ? new Date(destination.runtime.lastSuccessAt) : null;
|
||||
const lastSuccessMs = lastSuccessAt && Number.isFinite(lastSuccessAt.getTime())
|
||||
? lastSuccessAt.getTime()
|
||||
: Number.NEGATIVE_INFINITY;
|
||||
|
||||
const dayCursor = new Date(startMs);
|
||||
@@ -620,7 +686,7 @@ export function hasBackupSlotBetween(
|
||||
for (const slotStart of slotStarts) {
|
||||
const slotStartMs = slotStart.getTime();
|
||||
if (slotStartMs < startMs || slotStartMs >= endMs) continue;
|
||||
if (lastAttemptMs >= slotStartMs) continue;
|
||||
if (lastSuccessMs >= slotStartMs) continue;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -637,9 +703,9 @@ export function isBackupDueNow(
|
||||
): boolean {
|
||||
if (!destination.schedule.enabled) return false;
|
||||
const toleranceMs = Math.max(1, windowMinutes) * 60 * 1000;
|
||||
const lastAttemptAt = destination.runtime.lastAttemptAt ? new Date(destination.runtime.lastAttemptAt) : null;
|
||||
const lastAttemptMs = lastAttemptAt && Number.isFinite(lastAttemptAt.getTime())
|
||||
? lastAttemptAt.getTime()
|
||||
const lastSuccessAt = destination.runtime.lastSuccessAt ? new Date(destination.runtime.lastSuccessAt) : null;
|
||||
const lastSuccessMs = lastSuccessAt && Number.isFinite(lastSuccessAt.getTime())
|
||||
? lastSuccessAt.getTime()
|
||||
: Number.NEGATIVE_INFINITY;
|
||||
const localDateKey = getBackupLocalDateKey(now, destination.schedule.timezone);
|
||||
const slotStarts = getBackupSlotStartsForLocalDay(
|
||||
@@ -652,7 +718,7 @@ export function isBackupDueNow(
|
||||
for (const slotStart of slotStarts) {
|
||||
const slotStartMs = slotStart.getTime();
|
||||
if (now.getTime() < slotStartMs || now.getTime() >= slotStartMs + toleranceMs) continue;
|
||||
if (lastAttemptMs >= slotStartMs) return false;
|
||||
if (lastSuccessMs >= slotStartMs) return false;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
||||
@@ -33,6 +33,13 @@ export interface RemoteBackupFile {
|
||||
bytes: Uint8Array;
|
||||
}
|
||||
|
||||
export interface RemoteBackupFileStat {
|
||||
provider: BackupDestinationType;
|
||||
remotePath: string;
|
||||
size: number | null;
|
||||
modifiedAt: string | null;
|
||||
}
|
||||
|
||||
export interface RemoteBackupFilePutOptions {
|
||||
contentType?: string;
|
||||
}
|
||||
@@ -433,6 +440,10 @@ async function deleteFromWebDav(config: WebDavBackupDestination, relativePath: s
|
||||
}
|
||||
|
||||
async function existsInWebDav(config: WebDavBackupDestination, relativePath: string): Promise<boolean> {
|
||||
return (await statWebDavFile(config, relativePath)) !== null;
|
||||
}
|
||||
|
||||
async function statWebDavFile(config: WebDavBackupDestination, relativePath: string): Promise<RemoteBackupFileStat | null> {
|
||||
const authHeader = toBasicAuthHeader(config.username, config.password);
|
||||
const remotePath = webDavFullPath(config, relativePath);
|
||||
const response = await fetch(buildWebDavUrl(config.baseUrl, remotePath), {
|
||||
@@ -441,11 +452,17 @@ async function existsInWebDav(config: WebDavBackupDestination, relativePath: str
|
||||
Authorization: authHeader,
|
||||
},
|
||||
});
|
||||
if (response.status === 404) return false;
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) {
|
||||
throw new Error(`WebDAV existence check failed: ${response.status}`);
|
||||
}
|
||||
return true;
|
||||
const size = Number(response.headers.get('Content-Length') || '');
|
||||
return {
|
||||
provider: 'webdav',
|
||||
remotePath: normalizeRelativePath(relativePath),
|
||||
size: Number.isFinite(size) ? size : null,
|
||||
modifiedAt: parseHttpDate(response.headers.get('Last-Modified') || ''),
|
||||
};
|
||||
}
|
||||
|
||||
function isBucketHostedS3Endpoint(endpoint: URL, bucket: string): boolean {
|
||||
@@ -540,61 +557,68 @@ async function listS3Entries(config: S3BackupDestination, relativePath: string):
|
||||
const currentPath = normalizeRelativePath(relativePath);
|
||||
const targetPrefixBase = normalizeS3ObjectKey(config, currentPath);
|
||||
const targetPrefix = trimSlashes(targetPrefixBase) ? `${trimSlashes(targetPrefixBase)}/` : '';
|
||||
const url = s3BucketBaseUrl(config);
|
||||
url.searchParams.set('list-type', '2');
|
||||
url.searchParams.set('delimiter', '/');
|
||||
if (targetPrefix) url.searchParams.set('prefix', targetPrefix);
|
||||
|
||||
const response = await signedS3Request(config, 'GET', url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`S3 listing failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const xml = await response.text();
|
||||
const rootPrefix = trimSlashes(config.rootPath);
|
||||
const items: RemoteBackupItem[] = [];
|
||||
let continuationToken = '';
|
||||
|
||||
for (const prefix of extractXmlBlocks(xml, 'CommonPrefixes')) {
|
||||
const fullPrefix = trimSlashes(extractXmlFirst(prefix, 'Prefix') || '');
|
||||
if (!fullPrefix) continue;
|
||||
const relative = rootPrefix
|
||||
? fullPrefix === rootPrefix
|
||||
? ''
|
||||
: fullPrefix.startsWith(`${rootPrefix}/`)
|
||||
? fullPrefix.slice(rootPrefix.length + 1)
|
||||
do {
|
||||
const url = s3BucketBaseUrl(config);
|
||||
url.searchParams.set('list-type', '2');
|
||||
url.searchParams.set('delimiter', '/');
|
||||
if (targetPrefix) url.searchParams.set('prefix', targetPrefix);
|
||||
if (continuationToken) url.searchParams.set('continuation-token', continuationToken);
|
||||
|
||||
const response = await signedS3Request(config, 'GET', url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`S3 listing failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const xml = await response.text();
|
||||
|
||||
for (const prefix of extractXmlBlocks(xml, 'CommonPrefixes')) {
|
||||
const fullPrefix = trimSlashes(extractXmlFirst(prefix, 'Prefix') || '');
|
||||
if (!fullPrefix) continue;
|
||||
const relative = rootPrefix
|
||||
? fullPrefix === rootPrefix
|
||||
? ''
|
||||
: fullPrefix.startsWith(`${rootPrefix}/`)
|
||||
? fullPrefix.slice(rootPrefix.length + 1)
|
||||
: ''
|
||||
: fullPrefix;
|
||||
const normalizedRelative = trimSlashes(relative);
|
||||
if (!normalizedRelative) continue;
|
||||
const itemPath = normalizedRelative.replace(/\/+$/, '');
|
||||
if ((parentPath(itemPath) || '') !== currentPath) continue;
|
||||
items.push({
|
||||
path: itemPath,
|
||||
name: basename(itemPath) || itemPath,
|
||||
isDirectory: true,
|
||||
size: null,
|
||||
modifiedAt: null,
|
||||
});
|
||||
}
|
||||
|
||||
for (const content of extractXmlBlocks(xml, 'Contents')) {
|
||||
const fullKey = trimSlashes(extractXmlFirst(content, 'Key') || '');
|
||||
if (!fullKey || (targetPrefix && fullKey === trimSlashes(targetPrefix))) continue;
|
||||
const relative = rootPrefix
|
||||
? fullKey.startsWith(`${rootPrefix}/`)
|
||||
? fullKey.slice(rootPrefix.length + 1)
|
||||
: ''
|
||||
: fullPrefix;
|
||||
const normalizedRelative = trimSlashes(relative);
|
||||
if (!normalizedRelative) continue;
|
||||
const itemPath = normalizedRelative.replace(/\/+$/, '');
|
||||
if ((parentPath(itemPath) || '') !== currentPath) continue;
|
||||
items.push({
|
||||
path: itemPath,
|
||||
name: basename(itemPath) || itemPath,
|
||||
isDirectory: true,
|
||||
size: null,
|
||||
modifiedAt: null,
|
||||
});
|
||||
}
|
||||
: fullKey;
|
||||
const normalizedRelative = trimSlashes(relative);
|
||||
if (!normalizedRelative || (parentPath(normalizedRelative) || '') !== currentPath) continue;
|
||||
items.push({
|
||||
path: normalizedRelative,
|
||||
name: basename(normalizedRelative) || normalizedRelative,
|
||||
isDirectory: false,
|
||||
size: Number(extractXmlFirst(content, 'Size') || 0) || null,
|
||||
modifiedAt: parseHttpDate(extractXmlFirst(content, 'LastModified') || '') || null,
|
||||
});
|
||||
}
|
||||
|
||||
for (const content of extractXmlBlocks(xml, 'Contents')) {
|
||||
const fullKey = trimSlashes(extractXmlFirst(content, 'Key') || '');
|
||||
if (!fullKey || (targetPrefix && fullKey === trimSlashes(targetPrefix))) continue;
|
||||
const relative = rootPrefix
|
||||
? fullKey.startsWith(`${rootPrefix}/`)
|
||||
? fullKey.slice(rootPrefix.length + 1)
|
||||
: ''
|
||||
: fullKey;
|
||||
const normalizedRelative = trimSlashes(relative);
|
||||
if (!normalizedRelative || (parentPath(normalizedRelative) || '') !== currentPath) continue;
|
||||
items.push({
|
||||
path: normalizedRelative,
|
||||
name: basename(normalizedRelative) || normalizedRelative,
|
||||
isDirectory: false,
|
||||
size: Number(extractXmlFirst(content, 'Size') || 0) || null,
|
||||
modifiedAt: parseHttpDate(extractXmlFirst(content, 'LastModified') || '') || null,
|
||||
});
|
||||
}
|
||||
continuationToken = extractXmlFirst(xml, 'NextContinuationToken') || '';
|
||||
} while (continuationToken);
|
||||
|
||||
const deduped = new Map<string, RemoteBackupItem>();
|
||||
for (const item of items) deduped.set(`${item.isDirectory ? 'd' : 'f'}:${item.path}`, item);
|
||||
@@ -637,14 +661,24 @@ async function deleteFromS3(config: S3BackupDestination, relativePath: string):
|
||||
}
|
||||
|
||||
async function existsInS3(config: S3BackupDestination, relativePath: string): Promise<boolean> {
|
||||
return (await statS3File(config, relativePath)) !== null;
|
||||
}
|
||||
|
||||
async function statS3File(config: S3BackupDestination, relativePath: string): Promise<RemoteBackupFileStat | null> {
|
||||
const objectKey = normalizeS3ObjectKey(config, relativePath);
|
||||
const url = s3ObjectUrl(config, objectKey);
|
||||
const response = await signedS3Request(config, 'HEAD', url);
|
||||
if (response.status === 404) return false;
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) {
|
||||
throw new Error(`S3 existence check failed: ${response.status}`);
|
||||
}
|
||||
return true;
|
||||
const size = Number(response.headers.get('Content-Length') || '');
|
||||
return {
|
||||
provider: 's3',
|
||||
remotePath: normalizeRelativePath(relativePath),
|
||||
size: Number.isFinite(size) ? size : null,
|
||||
modifiedAt: parseHttpDate(response.headers.get('Last-Modified') || ''),
|
||||
};
|
||||
}
|
||||
|
||||
interface ConfiguredDestinationAdapter {
|
||||
@@ -656,6 +690,7 @@ interface ConfiguredDestinationAdapter {
|
||||
download: (config: WebDavBackupDestination | S3BackupDestination, relativePath: string) => Promise<RemoteBackupFile>;
|
||||
deleteFile: (config: WebDavBackupDestination | S3BackupDestination, relativePath: string) => Promise<void>;
|
||||
exists: (config: WebDavBackupDestination | S3BackupDestination, relativePath: string) => Promise<boolean>;
|
||||
stat: (config: WebDavBackupDestination | S3BackupDestination, relativePath: string) => Promise<RemoteBackupFileStat | null>;
|
||||
}
|
||||
|
||||
export interface RemoteBackupTransferSession {
|
||||
@@ -666,6 +701,7 @@ export interface RemoteBackupTransferSession {
|
||||
download(relativePath: string): Promise<RemoteBackupFile>;
|
||||
deleteFile(relativePath: string): Promise<void>;
|
||||
exists(relativePath: string): Promise<boolean>;
|
||||
stat(relativePath: string): Promise<RemoteBackupFileStat | null>;
|
||||
}
|
||||
|
||||
function resolveConfiguredDestinationAdapter(
|
||||
@@ -683,6 +719,7 @@ function resolveConfiguredDestinationAdapter(
|
||||
download: (config, relativePath) => downloadFromWebDav(config as WebDavBackupDestination, relativePath),
|
||||
deleteFile: (config, relativePath) => deleteFromWebDav(config as WebDavBackupDestination, relativePath),
|
||||
exists: (config, relativePath) => existsInWebDav(config as WebDavBackupDestination, relativePath),
|
||||
stat: (config, relativePath) => statWebDavFile(config as WebDavBackupDestination, relativePath),
|
||||
};
|
||||
}
|
||||
if (destination.type === 's3') {
|
||||
@@ -695,6 +732,7 @@ function resolveConfiguredDestinationAdapter(
|
||||
download: (config, relativePath) => downloadFromS3(config as S3BackupDestination, relativePath),
|
||||
deleteFile: (config, relativePath) => deleteFromS3(config as S3BackupDestination, relativePath),
|
||||
exists: (config, relativePath) => existsInS3(config as S3BackupDestination, relativePath),
|
||||
stat: (config, relativePath) => statS3File(config as S3BackupDestination, relativePath),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -730,6 +768,7 @@ export function createRemoteBackupTransferSession(destination: BackupDestination
|
||||
download: async (relativePath: string) => adapter.download(adapter.config, relativePath),
|
||||
deleteFile: async (relativePath: string) => adapter.deleteFile(adapter.config, normalizeRelativePath(relativePath)),
|
||||
exists: async (relativePath: string) => adapter.exists(adapter.config, normalizeRelativePath(relativePath)),
|
||||
stat: async (relativePath: string) => adapter.stat(adapter.config, normalizeRelativePath(relativePath)),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user