mirror of
https://github.com/shuaiplus/nodewarden.git
synced 2026-06-20 21:00:41 +00:00
feat: enhance backup functionality with attachment options
- Added support for including attachments in backup exports. - Updated backup-related interfaces and functions to handle attachment options. - Introduced a new UI component for selecting attachment inclusion during backup operations. - Modified existing components to integrate the new attachment functionality. - Improved user feedback and error handling during backup processes.
This commit is contained in:
+74
-205
@@ -2,9 +2,7 @@ import { zipSync, unzipSync } from 'fflate';
|
||||
import type { Env } from '../types';
|
||||
import {
|
||||
getAttachmentObjectKey,
|
||||
getBlobObject,
|
||||
getBlobStorageKind,
|
||||
getSendFileObjectKey,
|
||||
} from './blob-store';
|
||||
|
||||
type SqlRow = Record<string, string | number | null>;
|
||||
@@ -13,16 +11,12 @@ const BACKUP_FORMAT_VERSION = 1;
|
||||
const BACKUP_APP_VERSION = '1.3.0';
|
||||
// Worker-side backup export must stay well below Cloudflare CPU limits.
|
||||
// Prefer store-only ZIP entries over heavier compression to keep exports reliable.
|
||||
const BACKUP_TEXT_COMPRESSION_LEVEL = 1;
|
||||
const BACKUP_BINARY_COMPRESSION_LEVEL = 1;
|
||||
const BACKUP_R2_BLOB_READ_CONCURRENCY = 4;
|
||||
const BACKUP_KV_BLOB_READ_CONCURRENCY = 4;
|
||||
const BACKUP_R2_BLOB_READ_CHUNK_SIZE = 32;
|
||||
const BACKUP_KV_BLOB_READ_CHUNK_SIZE = 162;
|
||||
const MAX_BACKUP_ARCHIVE_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_BACKUP_ARCHIVE_ENTRY_COUNT = 800;
|
||||
const BACKUP_TEXT_COMPRESSION_LEVEL = 0;
|
||||
const BACKUP_JSON_INDENT = 2;
|
||||
const MAX_BACKUP_ARCHIVE_BYTES = 64 * 1024 * 1024;
|
||||
const MAX_BACKUP_ARCHIVE_ENTRY_COUNT = 10_000;
|
||||
const MAX_BACKUP_EXTRACTED_BYTES = 64 * 1024 * 1024;
|
||||
const MAX_BACKUP_DB_JSON_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_BACKUP_DB_JSON_BYTES = 32 * 1024 * 1024;
|
||||
|
||||
export interface BackupManifest {
|
||||
formatVersion: 1;
|
||||
@@ -32,14 +26,20 @@ export interface BackupManifest {
|
||||
tableCounts: Record<string, number>;
|
||||
includes: {
|
||||
attachments: boolean;
|
||||
sendFiles: boolean;
|
||||
};
|
||||
blobSummary: {
|
||||
attachmentFiles: number;
|
||||
sendFiles: number;
|
||||
totalBytes: number;
|
||||
largestObjectBytes: number;
|
||||
};
|
||||
attachmentBlobs?: BackupManifestAttachmentBlob[];
|
||||
}
|
||||
|
||||
export interface BackupManifestAttachmentBlob {
|
||||
cipherId: string;
|
||||
attachmentId: string;
|
||||
blobName: string;
|
||||
sizeBytes: number;
|
||||
}
|
||||
|
||||
export interface BackupPayload {
|
||||
@@ -51,7 +51,6 @@ export interface BackupPayload {
|
||||
folders: SqlRow[];
|
||||
ciphers: SqlRow[];
|
||||
attachments: SqlRow[];
|
||||
sends: SqlRow[];
|
||||
};
|
||||
}
|
||||
|
||||
@@ -61,27 +60,8 @@ export interface BackupArchiveBundle {
|
||||
manifest: BackupManifest;
|
||||
}
|
||||
|
||||
interface BackupBlobTask {
|
||||
archivePath: string;
|
||||
objectKey: string;
|
||||
kind: 'attachment' | 'send-file';
|
||||
missingMessage: string;
|
||||
}
|
||||
|
||||
interface BackupBlobTaskResult {
|
||||
archivePath: string;
|
||||
bytes: Uint8Array;
|
||||
kind: BackupBlobTask['kind'];
|
||||
}
|
||||
|
||||
export function parseSendFileId(data: string | null): string | null {
|
||||
if (!data) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(data) as Record<string, unknown>;
|
||||
return typeof parsed.id === 'string' && parsed.id.trim() ? parsed.id.trim() : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
export interface BuildBackupArchiveOptions {
|
||||
includeAttachments?: boolean;
|
||||
}
|
||||
|
||||
async function queryRows(db: D1Database, sql: string, ...values: unknown[]): Promise<SqlRow[]> {
|
||||
@@ -89,95 +69,6 @@ async function queryRows(db: D1Database, sql: string, ...values: unknown[]): Pro
|
||||
return (result.results || []).map((row) => ({ ...row }));
|
||||
}
|
||||
|
||||
async function streamToBytes(stream: ReadableStream | null): Promise<Uint8Array> {
|
||||
if (!stream) return new Uint8Array();
|
||||
const buffer = await new Response(stream).arrayBuffer();
|
||||
return new Uint8Array(buffer);
|
||||
}
|
||||
|
||||
function getBackupBlobReadConcurrency(env: Env): number {
|
||||
return getBlobStorageKind(env) === 'kv' ? BACKUP_KV_BLOB_READ_CONCURRENCY : BACKUP_R2_BLOB_READ_CONCURRENCY;
|
||||
}
|
||||
|
||||
function getBackupBlobReadChunkSize(env: Env): number {
|
||||
return getBlobStorageKind(env) === 'kv' ? BACKUP_KV_BLOB_READ_CHUNK_SIZE : BACKUP_R2_BLOB_READ_CHUNK_SIZE;
|
||||
}
|
||||
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: readonly T[],
|
||||
concurrency: number,
|
||||
worker: (item: T, index: number) => Promise<R>
|
||||
): Promise<R[]> {
|
||||
if (!items.length) return [];
|
||||
|
||||
const results = new Array<R>(items.length);
|
||||
let nextIndex = 0;
|
||||
|
||||
const runWorker = async (): Promise<void> => {
|
||||
while (true) {
|
||||
const currentIndex = nextIndex;
|
||||
nextIndex += 1;
|
||||
if (currentIndex >= items.length) return;
|
||||
results[currentIndex] = await worker(items[currentIndex], currentIndex);
|
||||
}
|
||||
};
|
||||
|
||||
const workerCount = Math.max(1, Math.min(concurrency, items.length));
|
||||
await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
|
||||
return results;
|
||||
}
|
||||
|
||||
async function loadBackupBlobFiles(env: Env, tasks: BackupBlobTask[]): Promise<{
|
||||
files: Record<string, Uint8Array>;
|
||||
attachmentFiles: number;
|
||||
sendFiles: number;
|
||||
totalBytes: number;
|
||||
largestObjectBytes: number;
|
||||
}> {
|
||||
const files: Record<string, Uint8Array> = {};
|
||||
let attachmentFiles = 0;
|
||||
let sendFiles = 0;
|
||||
let totalBytes = 0;
|
||||
let largestObjectBytes = 0;
|
||||
|
||||
const concurrency = getBackupBlobReadConcurrency(env);
|
||||
const chunkSize = getBackupBlobReadChunkSize(env);
|
||||
|
||||
for (let offset = 0; offset < tasks.length; offset += chunkSize) {
|
||||
const chunk = tasks.slice(offset, offset + chunkSize);
|
||||
const loaded = await mapWithConcurrency(chunk, concurrency, async (task) => {
|
||||
const object = await getBlobObject(env, task.objectKey);
|
||||
if (!object) {
|
||||
throw new Error(task.missingMessage);
|
||||
}
|
||||
return {
|
||||
archivePath: task.archivePath,
|
||||
bytes: await streamToBytes(object.body),
|
||||
kind: task.kind,
|
||||
} satisfies BackupBlobTaskResult;
|
||||
});
|
||||
|
||||
for (const item of loaded) {
|
||||
files[item.archivePath] = item.bytes;
|
||||
totalBytes += item.bytes.byteLength;
|
||||
largestObjectBytes = Math.max(largestObjectBytes, item.bytes.byteLength);
|
||||
if (item.kind === 'attachment') {
|
||||
attachmentFiles += 1;
|
||||
} else {
|
||||
sendFiles += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
files,
|
||||
attachmentFiles,
|
||||
sendFiles,
|
||||
totalBytes,
|
||||
largestObjectBytes,
|
||||
};
|
||||
}
|
||||
|
||||
function buildBackupFileName(date: Date = new Date()): string {
|
||||
const parts = [
|
||||
date.getUTCFullYear().toString().padStart(4, '0'),
|
||||
@@ -204,12 +95,6 @@ function getRequiredZipEntries(db: BackupPayload['db']): string[] {
|
||||
if (!cipherId || !attachmentId) continue;
|
||||
entries.push(`attachments/${cipherId}/${attachmentId}.bin`);
|
||||
}
|
||||
for (const row of db.sends) {
|
||||
const sendId = String(row.id || '').trim();
|
||||
const fileId = parseSendFileId(typeof row.data === 'string' ? row.data : null);
|
||||
if (!sendId || !fileId) continue;
|
||||
entries.push(`send-files/${sendId}/${fileId}.bin`);
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
@@ -223,13 +108,19 @@ function ensureRowArray(value: unknown, table: string): SqlRow[] {
|
||||
function createZipEntries(files: Record<string, Uint8Array>): Record<string, Uint8Array | [Uint8Array, { level: 0 | 1 | 6 }]> {
|
||||
const entries: Record<string, Uint8Array | [Uint8Array, { level: 0 | 1 | 6 }]> = {};
|
||||
for (const [path, bytes] of Object.entries(files)) {
|
||||
const isBinaryBlob = path.endsWith('.bin');
|
||||
entries[path] = [bytes, { level: isBinaryBlob ? BACKUP_BINARY_COMPRESSION_LEVEL : BACKUP_TEXT_COMPRESSION_LEVEL }];
|
||||
entries[path] = [bytes, { level: BACKUP_TEXT_COMPRESSION_LEVEL }];
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
export function parseBackupArchive(bytes: Uint8Array): { payload: BackupPayload; files: Record<string, Uint8Array> } {
|
||||
export interface ParseBackupArchiveOptions {
|
||||
allowExternalAttachmentBlobs?: boolean;
|
||||
}
|
||||
|
||||
export function parseBackupArchive(
|
||||
bytes: Uint8Array,
|
||||
options: ParseBackupArchiveOptions = {}
|
||||
): { payload: BackupPayload; files: Record<string, Uint8Array> } {
|
||||
validateArchiveSize(bytes);
|
||||
let zipped: Record<string, Uint8Array>;
|
||||
try {
|
||||
@@ -278,7 +169,12 @@ export function parseBackupArchive(bytes: Uint8Array): { payload: BackupPayload;
|
||||
throw new Error('Backup archive database payload is invalid');
|
||||
}
|
||||
|
||||
const requiredEntries = getRequiredZipEntries(db);
|
||||
const externalAttachmentKeys = new Set<string>(
|
||||
options.allowExternalAttachmentBlobs
|
||||
? (manifest.attachmentBlobs || []).map((item) => `attachments/${String(item.cipherId || '').trim()}/${String(item.attachmentId || '').trim()}.bin`)
|
||||
: []
|
||||
);
|
||||
const requiredEntries = getRequiredZipEntries(db).filter((entry) => !externalAttachmentKeys.has(entry));
|
||||
for (const entry of requiredEntries) {
|
||||
if (!zipped[entry]) {
|
||||
throw new Error(`Backup archive is missing required file: ${entry}`);
|
||||
@@ -291,14 +187,26 @@ export function parseBackupArchive(bytes: Uint8Array): { payload: BackupPayload;
|
||||
};
|
||||
}
|
||||
|
||||
export function validateBackupPayloadContents(payload: BackupPayload, files: Record<string, Uint8Array>): void {
|
||||
export interface ValidateBackupPayloadOptions {
|
||||
allowExternalAttachmentBlobs?: boolean;
|
||||
}
|
||||
|
||||
export function validateBackupPayloadContents(
|
||||
payload: BackupPayload,
|
||||
files: Record<string, Uint8Array>,
|
||||
options: ValidateBackupPayloadOptions = {}
|
||||
): void {
|
||||
const configRows = ensureRowArray(payload.db.config, 'config');
|
||||
const userRows = ensureRowArray(payload.db.users, 'users');
|
||||
const revisionRows = ensureRowArray(payload.db.user_revisions, 'user_revisions');
|
||||
const folderRows = ensureRowArray(payload.db.folders, 'folders');
|
||||
const cipherRows = ensureRowArray(payload.db.ciphers, 'ciphers');
|
||||
const attachmentRows = ensureRowArray(payload.db.attachments, 'attachments');
|
||||
const sendRows = ensureRowArray(payload.db.sends, 'sends');
|
||||
const externalAttachmentKeys = new Set<string>(
|
||||
options.allowExternalAttachmentBlobs
|
||||
? (payload.manifest.attachmentBlobs || []).map((item) => `attachments/${String(item.cipherId || '').trim()}/${String(item.attachmentId || '').trim()}.bin`)
|
||||
: []
|
||||
);
|
||||
|
||||
const userIds = new Set<string>();
|
||||
for (const row of userRows) {
|
||||
@@ -349,36 +257,39 @@ export function validateBackupPayloadContents(payload: BackupPayload, files: Rec
|
||||
if (!id || !cipherId || !cipherIds.has(cipherId)) {
|
||||
throw new Error('Backup archive contains an invalid attachment row');
|
||||
}
|
||||
if (!files[`attachments/${cipherId}/${id}.bin`]) {
|
||||
const attachmentPath = `attachments/${cipherId}/${id}.bin`;
|
||||
if (!files[attachmentPath] && !externalAttachmentKeys.has(attachmentPath)) {
|
||||
throw new Error(`Backup archive is missing required file: attachments/${cipherId}/${id}.bin`);
|
||||
}
|
||||
}
|
||||
|
||||
const sendIds = new Set<string>();
|
||||
for (const row of sendRows) {
|
||||
const id = String(row.id || '').trim();
|
||||
const userId = String(row.user_id || '').trim();
|
||||
if (!id || !userIds.has(userId)) throw new Error('Backup archive contains an invalid send row');
|
||||
if (sendIds.has(id)) throw new Error(`Backup archive contains duplicate send id: ${id}`);
|
||||
sendIds.add(id);
|
||||
const fileId = parseSendFileId(typeof row.data === 'string' ? row.data : null);
|
||||
if (fileId && !files[`send-files/${id}/${fileId}.bin`]) {
|
||||
throw new Error(`Backup archive is missing required file: send-files/${id}/${fileId}.bin`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildBackupArchive(env: Env, date: Date = new Date()): Promise<BackupArchiveBundle> {
|
||||
export async function buildBackupArchive(
|
||||
env: Env,
|
||||
date: Date = new Date(),
|
||||
options: BuildBackupArchiveOptions = {}
|
||||
): Promise<BackupArchiveBundle> {
|
||||
const encoder = new TextEncoder();
|
||||
const [configRows, userRows, revisionRows, folderRows, cipherRows, attachmentRows, sendRows] = await Promise.all([
|
||||
const [configRows, userRows, revisionRows, folderRows, cipherRows, attachmentRows] = await Promise.all([
|
||||
queryRows(env.DB, 'SELECT key, value FROM config ORDER BY key ASC'),
|
||||
queryRows(env.DB, 'SELECT id, email, name, master_password_hint, master_password_hash, key, private_key, public_key, kdf_type, kdf_iterations, kdf_memory, kdf_parallelism, security_stamp, role, status, totp_secret, totp_recovery_code, created_at, updated_at FROM users ORDER BY created_at ASC'),
|
||||
queryRows(env.DB, 'SELECT user_id, revision_date FROM user_revisions ORDER BY user_id ASC'),
|
||||
queryRows(env.DB, 'SELECT id, user_id, name, created_at, updated_at FROM folders ORDER BY created_at ASC'),
|
||||
queryRows(env.DB, 'SELECT id, user_id, type, folder_id, name, notes, favorite, data, reprompt, key, created_at, updated_at, deleted_at FROM ciphers ORDER BY created_at ASC'),
|
||||
queryRows(env.DB, 'SELECT id, cipher_id, file_name, size, size_name, key FROM attachments ORDER BY cipher_id ASC, id ASC'),
|
||||
queryRows(env.DB, 'SELECT id, user_id, type, name, notes, data, key, password_hash, password_salt, password_iterations, auth_type, emails, max_access_count, access_count, disabled, hide_email, created_at, updated_at, expiration_date, deletion_date FROM sends ORDER BY created_at ASC'),
|
||||
]);
|
||||
const includeAttachments = options.includeAttachments !== false;
|
||||
const exportedAttachmentRows = includeAttachments ? attachmentRows : [];
|
||||
const attachmentBlobs: BackupManifestAttachmentBlob[] = exportedAttachmentRows.map((row) => {
|
||||
const cipherId = String(row.cipher_id || '').trim();
|
||||
const attachmentId = String(row.id || '').trim();
|
||||
return {
|
||||
cipherId,
|
||||
attachmentId,
|
||||
blobName: getAttachmentObjectKey(cipherId, attachmentId),
|
||||
sizeBytes: Number(row.size || 0) || 0,
|
||||
};
|
||||
});
|
||||
|
||||
const manifestBase = {
|
||||
formatVersion: BACKUP_FORMAT_VERSION,
|
||||
@@ -391,76 +302,34 @@ export async function buildBackupArchive(env: Env, date: Date = new Date()): Pro
|
||||
user_revisions: revisionRows.length,
|
||||
folders: folderRows.length,
|
||||
ciphers: cipherRows.length,
|
||||
attachments: attachmentRows.length,
|
||||
sends: sendRows.length,
|
||||
attachments: exportedAttachmentRows.length,
|
||||
},
|
||||
includes: {
|
||||
attachments: true,
|
||||
sendFiles: true,
|
||||
attachments: includeAttachments,
|
||||
},
|
||||
blobSummary: {
|
||||
attachmentFiles: 0,
|
||||
sendFiles: 0,
|
||||
totalBytes: 0,
|
||||
largestObjectBytes: 0,
|
||||
attachmentFiles: attachmentBlobs.length,
|
||||
totalBytes: attachmentBlobs.reduce((sum, item) => sum + item.sizeBytes, 0),
|
||||
largestObjectBytes: attachmentBlobs.reduce((max, item) => Math.max(max, item.sizeBytes), 0),
|
||||
},
|
||||
attachmentBlobs: includeAttachments ? attachmentBlobs : [],
|
||||
} satisfies BackupManifest;
|
||||
|
||||
const files: Record<string, Uint8Array> = {
|
||||
'manifest.json': encoder.encode(JSON.stringify(manifestBase)),
|
||||
'manifest.json': encoder.encode(JSON.stringify(manifestBase, null, BACKUP_JSON_INDENT)),
|
||||
'db.json': encoder.encode(JSON.stringify({
|
||||
config: configRows,
|
||||
users: userRows,
|
||||
user_revisions: revisionRows,
|
||||
folders: folderRows,
|
||||
ciphers: cipherRows,
|
||||
attachments: attachmentRows,
|
||||
sends: sendRows,
|
||||
})),
|
||||
attachments: exportedAttachmentRows,
|
||||
}, null, BACKUP_JSON_INDENT)),
|
||||
};
|
||||
|
||||
const blobTasks: BackupBlobTask[] = [];
|
||||
for (const row of attachmentRows) {
|
||||
const cipherId = String(row.cipher_id || '').trim();
|
||||
const attachmentId = String(row.id || '').trim();
|
||||
if (!cipherId || !attachmentId) continue;
|
||||
blobTasks.push({
|
||||
archivePath: `attachments/${cipherId}/${attachmentId}.bin`,
|
||||
objectKey: getAttachmentObjectKey(cipherId, attachmentId),
|
||||
kind: 'attachment',
|
||||
missingMessage: `Attachment blob missing for ${cipherId}/${attachmentId}`,
|
||||
});
|
||||
}
|
||||
|
||||
for (const row of sendRows) {
|
||||
const sendId = String(row.id || '').trim();
|
||||
const fileId = parseSendFileId(typeof row.data === 'string' ? row.data : null);
|
||||
if (!sendId || !fileId) continue;
|
||||
blobTasks.push({
|
||||
archivePath: `send-files/${sendId}/${fileId}.bin`,
|
||||
objectKey: getSendFileObjectKey(sendId, fileId),
|
||||
kind: 'send-file',
|
||||
missingMessage: `Send file blob missing for ${sendId}/${fileId}`,
|
||||
});
|
||||
}
|
||||
|
||||
const blobFiles = await loadBackupBlobFiles(env, blobTasks);
|
||||
Object.assign(files, blobFiles.files);
|
||||
|
||||
const manifest: BackupManifest = {
|
||||
...manifestBase,
|
||||
blobSummary: {
|
||||
attachmentFiles: blobFiles.attachmentFiles,
|
||||
sendFiles: blobFiles.sendFiles,
|
||||
totalBytes: blobFiles.totalBytes,
|
||||
largestObjectBytes: blobFiles.largestObjectBytes,
|
||||
},
|
||||
};
|
||||
files['manifest.json'] = encoder.encode(JSON.stringify(manifest));
|
||||
|
||||
return {
|
||||
bytes: zipSync(createZipEntries(files)),
|
||||
fileName: buildBackupFileName(date),
|
||||
manifest,
|
||||
manifest: manifestBase,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -264,6 +264,9 @@ function normalizeDestinationRecord(
|
||||
id,
|
||||
name,
|
||||
type,
|
||||
includeAttachments: typeof input.includeAttachments === 'boolean'
|
||||
? input.includeAttachments
|
||||
: previous?.includeAttachments ?? false,
|
||||
destination,
|
||||
schedule,
|
||||
runtime,
|
||||
@@ -280,6 +283,7 @@ function parseLegacyBackupSettings(rawValue: Record<string, unknown>, fallbackTi
|
||||
id: createBackupRandomId(),
|
||||
name: defaultDestinationName(destinationType, 1),
|
||||
type: destinationType,
|
||||
includeAttachments: false,
|
||||
destination: normalizeDestination(destinationType, rawValue.destination),
|
||||
schedule: {
|
||||
enabled: !!rawValue.enabled,
|
||||
|
||||
+298
-95
@@ -1,8 +1,13 @@
|
||||
import type { Env } from '../types';
|
||||
import { StorageService } from './storage';
|
||||
import { KV_MAX_OBJECT_BYTES, deleteBlobObject, getAttachmentObjectKey, getBlobStorageKind, getSendFileObjectKey, putBlobObject } from './blob-store';
|
||||
import { KV_MAX_OBJECT_BYTES, deleteBlobObject, getAttachmentObjectKey, getBlobStorageKind, putBlobObject } from './blob-store';
|
||||
import { normalizeImportedBackupSettings } from './backup-config';
|
||||
import { type BackupPayload, parseBackupArchive, parseSendFileId, validateBackupPayloadContents } from './backup-archive';
|
||||
import {
|
||||
type BackupManifestAttachmentBlob,
|
||||
type BackupPayload,
|
||||
parseBackupArchive,
|
||||
validateBackupPayloadContents,
|
||||
} from './backup-archive';
|
||||
|
||||
type SqlRow = Record<string, string | number | null>;
|
||||
|
||||
@@ -15,16 +20,13 @@ export interface BackupImportResultBody {
|
||||
folders: number;
|
||||
ciphers: number;
|
||||
attachments: number;
|
||||
sends: number;
|
||||
attachmentFiles: number;
|
||||
sendFiles: number;
|
||||
};
|
||||
skipped: {
|
||||
reason: string | null;
|
||||
attachments: number;
|
||||
sendFiles: number;
|
||||
items: Array<{
|
||||
kind: 'attachment' | 'send-file';
|
||||
kind: 'attachment';
|
||||
path: string;
|
||||
sizeBytes: number;
|
||||
}>;
|
||||
@@ -88,42 +90,18 @@ async function collectCurrentBlobKeys(db: D1Database): Promise<Set<string>> {
|
||||
if (!cipherId || !attachmentId) continue;
|
||||
keys.add(getAttachmentObjectKey(cipherId, attachmentId));
|
||||
}
|
||||
|
||||
const sendRows = await queryRows(db, 'SELECT id, data FROM sends');
|
||||
for (const row of sendRows) {
|
||||
const sendId = String(row.id || '').trim();
|
||||
const fileId = parseSendFileId(typeof row.data === 'string' ? row.data : null);
|
||||
if (!sendId || !fileId) continue;
|
||||
keys.add(getSendFileObjectKey(sendId, fileId));
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
function collectImportedBlobKeys(db: BackupPayload['db']): Set<string> {
|
||||
const keys = new Set<string>();
|
||||
for (const row of db.attachments) {
|
||||
const cipherId = String(row.cipher_id || '').trim();
|
||||
const attachmentId = String(row.id || '').trim();
|
||||
if (!cipherId || !attachmentId) continue;
|
||||
keys.add(getAttachmentObjectKey(cipherId, attachmentId));
|
||||
}
|
||||
for (const row of db.sends) {
|
||||
const sendId = String(row.id || '').trim();
|
||||
const fileId = parseSendFileId(typeof row.data === 'string' ? row.data : null);
|
||||
if (!sendId || !fileId) continue;
|
||||
keys.add(getSendFileObjectKey(sendId, fileId));
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
const KV_BLOB_SKIP_REASON = 'Cloudflare KV object size limit (25 MB)';
|
||||
const BLOB_STORAGE_UNAVAILABLE_SKIP_REASON = 'Attachment storage is not configured';
|
||||
const ATTACHMENT_RESTORE_FAILED_REASON = 'Some attachments could not be restored and were skipped';
|
||||
|
||||
interface BackupImportSkipSummary {
|
||||
reason: string | null;
|
||||
attachments: number;
|
||||
sendFiles: number;
|
||||
items: Array<{
|
||||
kind: 'attachment' | 'send-file';
|
||||
kind: 'attachment';
|
||||
path: string;
|
||||
sizeBytes: number;
|
||||
}>;
|
||||
@@ -134,21 +112,58 @@ interface PreparedBackupImportPayload {
|
||||
skipped: BackupImportSkipSummary;
|
||||
}
|
||||
|
||||
interface AttachmentRestoreResult {
|
||||
imported: number;
|
||||
restoredAttachments: SqlRow[];
|
||||
skipped: BackupImportSkipSummary;
|
||||
}
|
||||
|
||||
interface RemoteAttachmentSource {
|
||||
hasAttachment(blobName: string): Promise<boolean>;
|
||||
loadAttachment(blobName: string): Promise<Uint8Array | null>;
|
||||
}
|
||||
|
||||
function prepareImportPayloadForTarget(env: Env, payload: BackupPayload, files: Record<string, Uint8Array>): PreparedBackupImportPayload {
|
||||
if (getBlobStorageKind(env) !== 'kv') {
|
||||
const storageKind = getBlobStorageKind(env);
|
||||
if (storageKind === 'r2') {
|
||||
return {
|
||||
payload,
|
||||
skipped: {
|
||||
reason: null,
|
||||
attachments: 0,
|
||||
sendFiles: 0,
|
||||
items: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (storageKind === null) {
|
||||
const skippedItems = (payload.db.attachments || []).map((row) => {
|
||||
const cipherId = String(row.cipher_id || '').trim();
|
||||
const attachmentId = String(row.id || '').trim();
|
||||
return {
|
||||
kind: 'attachment' as const,
|
||||
path: `attachments/${cipherId}/${attachmentId}.bin`,
|
||||
sizeBytes: Number(row.size || 0) || 0,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
payload: {
|
||||
...payload,
|
||||
db: {
|
||||
...payload.db,
|
||||
attachments: [],
|
||||
},
|
||||
},
|
||||
skipped: {
|
||||
reason: skippedItems.length ? BLOB_STORAGE_UNAVAILABLE_SKIP_REASON : null,
|
||||
attachments: skippedItems.length,
|
||||
items: skippedItems,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const oversizedAttachmentPaths = new Set<string>();
|
||||
const oversizedSendPaths = new Set<string>();
|
||||
const skippedItems: BackupImportSkipSummary['items'] = [];
|
||||
|
||||
for (const entry of Object.keys(files)) {
|
||||
@@ -158,11 +173,6 @@ function prepareImportPayloadForTarget(env: Env, payload: BackupPayload, files:
|
||||
if (entry.startsWith('attachments/')) {
|
||||
oversizedAttachmentPaths.add(entry);
|
||||
skippedItems.push({ kind: 'attachment', path: entry, sizeBytes });
|
||||
continue;
|
||||
}
|
||||
if (entry.startsWith('send-files/')) {
|
||||
oversizedSendPaths.add(entry);
|
||||
skippedItems.push({ kind: 'send-file', path: entry, sizeBytes });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -173,28 +183,15 @@ function prepareImportPayloadForTarget(env: Env, payload: BackupPayload, files:
|
||||
return !oversizedAttachmentPaths.has(`attachments/${cipherId}/${attachmentId}.bin`);
|
||||
});
|
||||
|
||||
const nextSends = (payload.db.sends || []).filter((row) => {
|
||||
const sendId = String(row.id || '').trim();
|
||||
const fileId = parseSendFileId(typeof row.data === 'string' ? row.data : null);
|
||||
if (!sendId || !fileId) return true;
|
||||
return !oversizedSendPaths.has(`send-files/${sendId}/${fileId}.bin`);
|
||||
});
|
||||
|
||||
const nextPayload: BackupPayload = {
|
||||
...payload,
|
||||
db: {
|
||||
...payload.db,
|
||||
attachments: nextAttachments,
|
||||
sends: nextSends,
|
||||
},
|
||||
};
|
||||
|
||||
const needsKvBlobStorage = nextAttachments.length > 0
|
||||
|| nextSends.some((row) => {
|
||||
const sendId = String(row.id || '').trim();
|
||||
const fileId = parseSendFileId(typeof row.data === 'string' ? row.data : null);
|
||||
return !!sendId && !!fileId;
|
||||
});
|
||||
const needsKvBlobStorage = nextAttachments.length > 0;
|
||||
|
||||
if (needsKvBlobStorage && !env.ATTACHMENTS_KV) {
|
||||
throw new Error('Backup restore requires ATTACHMENTS_KV when using KV blob storage');
|
||||
@@ -204,8 +201,7 @@ function prepareImportPayloadForTarget(env: Env, payload: BackupPayload, files:
|
||||
payload: nextPayload,
|
||||
skipped: {
|
||||
reason: skippedItems.length ? KV_BLOB_SKIP_REASON : null,
|
||||
attachments: skippedItems.filter((item) => item.kind === 'attachment').length,
|
||||
sendFiles: skippedItems.filter((item) => item.kind === 'send-file').length,
|
||||
attachments: skippedItems.length,
|
||||
items: skippedItems,
|
||||
},
|
||||
};
|
||||
@@ -218,9 +214,9 @@ function buildInsertStatements(db: D1Database, table: string, columns: string[],
|
||||
return rows.map((row) => db.prepare(sql).bind(...columns.map((column) => row[column] ?? null)));
|
||||
}
|
||||
|
||||
async function restoreBlobFiles(env: Env, db: BackupPayload['db'], files: Record<string, Uint8Array>): Promise<{ attachments: number; sendFiles: number }> {
|
||||
let attachmentCount = 0;
|
||||
let sendFileCount = 0;
|
||||
async function restoreBlobFiles(env: Env, db: BackupPayload['db'], files: Record<string, Uint8Array>): Promise<AttachmentRestoreResult> {
|
||||
const restoredAttachments: SqlRow[] = [];
|
||||
const skippedItems: BackupImportSkipSummary['items'] = [];
|
||||
|
||||
for (const row of db.attachments || []) {
|
||||
const cipherId = String(row.cipher_id || '').trim();
|
||||
@@ -228,31 +224,178 @@ async function restoreBlobFiles(env: Env, db: BackupPayload['db'], files: Record
|
||||
if (!cipherId || !attachmentId) continue;
|
||||
const key = `attachments/${cipherId}/${attachmentId}.bin`;
|
||||
const bytes = files[key];
|
||||
if (!bytes) throw new Error(`Backup archive is missing required file: ${key}`);
|
||||
await putBlobObject(env, getAttachmentObjectKey(cipherId, attachmentId), bytes, {
|
||||
size: bytes.byteLength,
|
||||
contentType: 'application/octet-stream',
|
||||
});
|
||||
attachmentCount += 1;
|
||||
}
|
||||
|
||||
for (const row of db.sends || []) {
|
||||
const sendId = String(row.id || '').trim();
|
||||
const fileId = parseSendFileId(typeof row.data === 'string' ? row.data : null);
|
||||
if (!sendId || !fileId) continue;
|
||||
const key = `send-files/${sendId}/${fileId}.bin`;
|
||||
const bytes = files[key];
|
||||
if (!bytes) throw new Error(`Backup archive is missing required file: ${key}`);
|
||||
await putBlobObject(env, getSendFileObjectKey(sendId, fileId), bytes, {
|
||||
size: bytes.byteLength,
|
||||
contentType: 'application/octet-stream',
|
||||
});
|
||||
sendFileCount += 1;
|
||||
if (!bytes) {
|
||||
skippedItems.push({
|
||||
kind: 'attachment',
|
||||
path: key,
|
||||
sizeBytes: Number(row.size || 0) || 0,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await putBlobObject(env, getAttachmentObjectKey(cipherId, attachmentId), bytes, {
|
||||
size: bytes.byteLength,
|
||||
contentType: 'application/octet-stream',
|
||||
});
|
||||
restoredAttachments.push(row);
|
||||
} catch {
|
||||
skippedItems.push({
|
||||
kind: 'attachment',
|
||||
path: key,
|
||||
sizeBytes: bytes.byteLength,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
attachments: attachmentCount,
|
||||
sendFiles: sendFileCount,
|
||||
imported: restoredAttachments.length,
|
||||
restoredAttachments,
|
||||
skipped: {
|
||||
reason: skippedItems.length ? ATTACHMENT_RESTORE_FAILED_REASON : null,
|
||||
attachments: skippedItems.length,
|
||||
items: skippedItems,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildAttachmentBlobLookup(manifest: BackupPayload['manifest']): Map<string, BackupManifestAttachmentBlob> {
|
||||
return new Map(
|
||||
(manifest.attachmentBlobs || []).map((item) => [`${item.cipherId}/${item.attachmentId}`, item])
|
||||
);
|
||||
}
|
||||
|
||||
async function prepareRemoteAttachmentPayload(
|
||||
env: Env,
|
||||
payload: BackupPayload,
|
||||
files: Record<string, Uint8Array>,
|
||||
source: RemoteAttachmentSource
|
||||
): Promise<PreparedBackupImportPayload> {
|
||||
const manifestLookup = buildAttachmentBlobLookup(payload.manifest);
|
||||
const storageKind = getBlobStorageKind(env);
|
||||
const nextAttachments: SqlRow[] = [];
|
||||
const skippedItems: BackupImportSkipSummary['items'] = [];
|
||||
|
||||
for (const row of payload.db.attachments || []) {
|
||||
const cipherId = String(row.cipher_id || '').trim();
|
||||
const attachmentId = String(row.id || '').trim();
|
||||
const lookupKey = `${cipherId}/${attachmentId}`;
|
||||
const ref = manifestLookup.get(lookupKey);
|
||||
const sizeBytes = ref?.sizeBytes || Number(row.size || 0) || 0;
|
||||
const path = ref ? `attachments/${ref.blobName}` : `attachments/${lookupKey}`;
|
||||
const inlinePath = `attachments/${cipherId}/${attachmentId}.bin`;
|
||||
|
||||
if (files[inlinePath]) {
|
||||
nextAttachments.push(row);
|
||||
continue;
|
||||
}
|
||||
if (!ref) {
|
||||
skippedItems.push({ kind: 'attachment', path, sizeBytes });
|
||||
continue;
|
||||
}
|
||||
if (storageKind === 'kv' && sizeBytes > KV_MAX_OBJECT_BYTES) {
|
||||
skippedItems.push({ kind: 'attachment', path, sizeBytes });
|
||||
continue;
|
||||
}
|
||||
if (storageKind === null) {
|
||||
skippedItems.push({ kind: 'attachment', path, sizeBytes });
|
||||
continue;
|
||||
}
|
||||
if (!(await source.hasAttachment(ref.blobName))) {
|
||||
skippedItems.push({ kind: 'attachment', path, sizeBytes });
|
||||
continue;
|
||||
}
|
||||
nextAttachments.push(row);
|
||||
}
|
||||
|
||||
return {
|
||||
payload: {
|
||||
...payload,
|
||||
db: {
|
||||
...payload.db,
|
||||
attachments: nextAttachments,
|
||||
},
|
||||
},
|
||||
skipped: {
|
||||
reason: skippedItems.length ? 'Some remote attachments were unavailable and were skipped' : null,
|
||||
attachments: skippedItems.length,
|
||||
items: skippedItems,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function removeAttachmentRows(db: D1Database, attachmentRows: SqlRow[]): Promise<void> {
|
||||
if (!attachmentRows.length) return;
|
||||
const statements = attachmentRows
|
||||
.map((row) => {
|
||||
const attachmentId = String(row.id || '').trim();
|
||||
const cipherId = String(row.cipher_id || '').trim();
|
||||
if (!attachmentId || !cipherId) return null;
|
||||
return db.prepare('DELETE FROM attachments WHERE id = ? AND cipher_id = ?').bind(attachmentId, cipherId);
|
||||
})
|
||||
.filter((statement): statement is D1PreparedStatement => !!statement);
|
||||
if (!statements.length) return;
|
||||
await db.batch(statements);
|
||||
}
|
||||
|
||||
async function restoreRemoteAttachmentFiles(
|
||||
env: Env,
|
||||
payload: BackupPayload,
|
||||
files: Record<string, Uint8Array>,
|
||||
source: RemoteAttachmentSource
|
||||
): Promise<{
|
||||
imported: number;
|
||||
skipped: BackupImportSkipSummary;
|
||||
restoredAttachments: SqlRow[];
|
||||
}> {
|
||||
const manifestLookup = buildAttachmentBlobLookup(payload.manifest);
|
||||
const restoredAttachments: SqlRow[] = [];
|
||||
const skippedItems: BackupImportSkipSummary['items'] = [];
|
||||
|
||||
for (const row of payload.db.attachments || []) {
|
||||
const cipherId = String(row.cipher_id || '').trim();
|
||||
const attachmentId = String(row.id || '').trim();
|
||||
const inlinePath = `attachments/${cipherId}/${attachmentId}.bin`;
|
||||
const ref = manifestLookup.get(`${cipherId}/${attachmentId}`);
|
||||
if (!ref && !files[inlinePath]) {
|
||||
skippedItems.push({
|
||||
kind: 'attachment',
|
||||
path: `attachments/${cipherId}/${attachmentId}`,
|
||||
sizeBytes: Number(row.size || 0) || 0,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const bytes = files[inlinePath] || (ref ? await source.loadAttachment(ref.blobName) : null);
|
||||
if (!bytes) {
|
||||
skippedItems.push({
|
||||
kind: 'attachment',
|
||||
path: ref ? `attachments/${ref.blobName}` : inlinePath,
|
||||
sizeBytes: ref?.sizeBytes || Number(row.size || 0) || 0,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await putBlobObject(env, getAttachmentObjectKey(cipherId, attachmentId), bytes, {
|
||||
size: bytes.byteLength,
|
||||
contentType: 'application/octet-stream',
|
||||
});
|
||||
restoredAttachments.push(row);
|
||||
} catch {
|
||||
skippedItems.push({
|
||||
kind: 'attachment',
|
||||
path: ref ? `attachments/${ref.blobName}` : inlinePath,
|
||||
sizeBytes: bytes.byteLength,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
imported: restoredAttachments.length,
|
||||
restoredAttachments,
|
||||
skipped: {
|
||||
reason: skippedItems.length ? ATTACHMENT_RESTORE_FAILED_REASON : null,
|
||||
attachments: skippedItems.length,
|
||||
items: skippedItems,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -282,12 +425,6 @@ async function importBackupRows(db: D1Database, payload: BackupPayload['db']): P
|
||||
payload.ciphers || []
|
||||
),
|
||||
...buildInsertStatements(db, 'attachments', ['id', 'cipher_id', 'file_name', 'size', 'size_name', 'key'], payload.attachments || []),
|
||||
...buildInsertStatements(
|
||||
db,
|
||||
'sends',
|
||||
['id', 'user_id', 'type', 'name', 'notes', 'data', 'key', 'password_hash', 'password_salt', 'password_iterations', 'auth_type', 'emails', 'max_access_count', 'access_count', 'disabled', 'hide_email', 'created_at', 'updated_at', 'expiration_date', 'deletion_date'],
|
||||
payload.sends || []
|
||||
),
|
||||
];
|
||||
await db.batch(statements);
|
||||
}
|
||||
@@ -316,9 +453,11 @@ export async function importBackupArchiveBytes(
|
||||
await importBackupRows(env.DB, db);
|
||||
await normalizeImportedBackupSettings(storage, env, 'UTC');
|
||||
|
||||
const blobCounts = await restoreBlobFiles(env, db, parsed.files);
|
||||
const restored = await restoreBlobFiles(env, db, parsed.files);
|
||||
const failedRestoreRows = (db.attachments || []).filter((row) => !restored.restoredAttachments.includes(row));
|
||||
await removeAttachmentRows(env.DB, failedRestoreRows);
|
||||
if (replaceExisting && previousBlobKeys.size) {
|
||||
await cleanupOrphanedBlobFiles(env, previousBlobKeys, collectImportedBlobKeys(db));
|
||||
await cleanupOrphanedBlobFiles(env, previousBlobKeys, await collectCurrentBlobKeys(env.DB));
|
||||
}
|
||||
|
||||
await storage.setRegistered();
|
||||
@@ -333,12 +472,76 @@ export async function importBackupArchiveBytes(
|
||||
userRevisions: (db.user_revisions || []).length,
|
||||
folders: (db.folders || []).length,
|
||||
ciphers: (db.ciphers || []).length,
|
||||
attachments: (db.attachments || []).length,
|
||||
sends: (db.sends || []).length,
|
||||
attachmentFiles: blobCounts.attachments,
|
||||
sendFiles: blobCounts.sendFiles,
|
||||
attachments: restored.restoredAttachments.length,
|
||||
attachmentFiles: restored.imported,
|
||||
},
|
||||
skipped: {
|
||||
reason: restored.skipped.reason || prepared.skipped.reason,
|
||||
attachments: prepared.skipped.attachments + restored.skipped.attachments,
|
||||
items: [...prepared.skipped.items, ...restored.skipped.items],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function importRemoteBackupArchiveBytes(
|
||||
archiveBytes: Uint8Array,
|
||||
env: Env,
|
||||
actorUserId: string,
|
||||
replaceExisting: boolean,
|
||||
source: RemoteAttachmentSource
|
||||
): Promise<BackupImportExecutionResult> {
|
||||
const storage = new StorageService(env.DB);
|
||||
const parsed = parseBackupArchive(archiveBytes, { allowExternalAttachmentBlobs: true });
|
||||
const preparedRemote = await prepareRemoteAttachmentPayload(env, parsed.payload, parsed.files, source);
|
||||
validateBackupPayloadContents(preparedRemote.payload, parsed.files, { allowExternalAttachmentBlobs: true });
|
||||
|
||||
try {
|
||||
await ensureImportTargetIsFresh(env.DB);
|
||||
} catch (error) {
|
||||
if (!replaceExisting) {
|
||||
throw error instanceof Error ? error : new Error('Backup import requires a fresh instance');
|
||||
}
|
||||
}
|
||||
|
||||
const previousBlobKeys = replaceExisting ? await collectCurrentBlobKeys(env.DB) : new Set<string>();
|
||||
const { db } = preparedRemote.payload;
|
||||
await importBackupRows(env.DB, db);
|
||||
await normalizeImportedBackupSettings(storage, env, 'UTC');
|
||||
|
||||
const restored = await restoreRemoteAttachmentFiles(env, preparedRemote.payload, parsed.files, source);
|
||||
const failedRestoreRows = (db.attachments || []).filter((row) => !restored.restoredAttachments.includes(row));
|
||||
await removeAttachmentRows(env.DB, failedRestoreRows);
|
||||
|
||||
if (replaceExisting && previousBlobKeys.size) {
|
||||
await cleanupOrphanedBlobFiles(env, previousBlobKeys, await collectCurrentBlobKeys(env.DB));
|
||||
}
|
||||
|
||||
await storage.setRegistered();
|
||||
|
||||
const finalSkippedItems = [...preparedRemote.skipped.items, ...restored.skipped.items];
|
||||
const finalSkippedReason = finalSkippedItems.length
|
||||
? restored.skipped.reason || preparedRemote.skipped.reason
|
||||
: null;
|
||||
|
||||
return {
|
||||
auditActorUserId: (db.users || []).some((row) => String(row.id || '').trim() === actorUserId) ? actorUserId : null,
|
||||
result: {
|
||||
object: 'instance-backup-import',
|
||||
imported: {
|
||||
config: (db.config || []).length,
|
||||
users: (db.users || []).length,
|
||||
userRevisions: (db.user_revisions || []).length,
|
||||
folders: (db.folders || []).length,
|
||||
ciphers: (db.ciphers || []).length,
|
||||
attachments: restored.restoredAttachments.length,
|
||||
attachmentFiles: restored.imported,
|
||||
},
|
||||
skipped: {
|
||||
reason: finalSkippedReason,
|
||||
attachments: finalSkippedItems.length,
|
||||
items: finalSkippedItems,
|
||||
},
|
||||
skipped: prepared.skipped,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -33,6 +33,10 @@ export interface RemoteBackupFile {
|
||||
bytes: Uint8Array;
|
||||
}
|
||||
|
||||
export interface RemoteBackupFilePutOptions {
|
||||
contentType?: string;
|
||||
}
|
||||
|
||||
function isBackupArchiveName(name: string): boolean {
|
||||
return /\.zip$/i.test(String(name || '').trim());
|
||||
}
|
||||
@@ -83,6 +87,9 @@ function parentPath(path: string): string | null {
|
||||
|
||||
function sortRemoteItems(items: RemoteBackupItem[]): RemoteBackupItem[] {
|
||||
return items.slice().sort((a, b) => {
|
||||
const aIsAttachmentsDir = a.isDirectory && a.name === 'attachments';
|
||||
const bIsAttachmentsDir = b.isDirectory && b.name === 'attachments';
|
||||
if (aIsAttachmentsDir !== bIsAttachmentsDir) return aIsAttachmentsDir ? -1 : 1;
|
||||
if (a.isDirectory !== b.isDirectory) return a.isDirectory ? -1 : 1;
|
||||
return a.name.localeCompare(b.name, 'en');
|
||||
});
|
||||
@@ -243,9 +250,14 @@ async function ensureWebDavDirectory(baseUrl: string, directoryPath: string, aut
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadToWebDav(config: WebDavBackupDestination, archive: Uint8Array, fileName: string): Promise<BackupUploadResult> {
|
||||
async function putToWebDav(
|
||||
config: WebDavBackupDestination,
|
||||
relativePath: string,
|
||||
bytes: Uint8Array,
|
||||
options: RemoteBackupFilePutOptions = {}
|
||||
): Promise<void> {
|
||||
const authHeader = toBasicAuthHeader(config.username, config.password);
|
||||
const remoteFilePath = buildJoinedPath(config.remotePath, fileName);
|
||||
const remoteFilePath = buildJoinedPath(config.remotePath, relativePath);
|
||||
const remoteDir = parentPath(remoteFilePath);
|
||||
|
||||
if (remoteDir) {
|
||||
@@ -256,19 +268,22 @@ async function uploadToWebDav(config: WebDavBackupDestination, archive: Uint8Arr
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
Authorization: authHeader,
|
||||
'Content-Type': 'application/zip',
|
||||
'Content-Length': String(archive.byteLength),
|
||||
'Content-Type': options.contentType || 'application/octet-stream',
|
||||
'Content-Length': String(bytes.byteLength),
|
||||
},
|
||||
body: archive,
|
||||
body: bytes,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`WebDAV upload failed: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadToWebDav(config: WebDavBackupDestination, archive: Uint8Array, fileName: string): Promise<BackupUploadResult> {
|
||||
await putToWebDav(config, fileName, archive, { contentType: 'application/zip' });
|
||||
return {
|
||||
provider: 'webdav',
|
||||
remotePath: remoteFilePath,
|
||||
remotePath: buildJoinedPath(config.remotePath, fileName),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -386,6 +401,22 @@ async function deleteFromWebDav(config: WebDavBackupDestination, relativePath: s
|
||||
}
|
||||
}
|
||||
|
||||
async function existsInWebDav(config: WebDavBackupDestination, relativePath: string): Promise<boolean> {
|
||||
const authHeader = toBasicAuthHeader(config.username, config.password);
|
||||
const remotePath = webDavFullPath(config, relativePath);
|
||||
const response = await fetch(buildWebDavUrl(config.baseUrl, remotePath), {
|
||||
method: 'HEAD',
|
||||
headers: {
|
||||
Authorization: authHeader,
|
||||
},
|
||||
});
|
||||
if (response.status === 404) return false;
|
||||
if (!response.ok) {
|
||||
throw new Error(`WebDAV existence check failed: ${response.status}`);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function e3BucketBaseUrl(config: E3BackupDestination): URL {
|
||||
return new URL(`${config.endpoint.replace(/\/+$/, '')}/${encodeURIComponent(config.bucket)}`);
|
||||
}
|
||||
@@ -396,9 +427,10 @@ function normalizeE3ObjectKey(config: E3BackupDestination, relativePath: string)
|
||||
|
||||
async function signedE3Request(
|
||||
config: E3BackupDestination,
|
||||
method: 'GET' | 'PUT' | 'DELETE',
|
||||
method: 'GET' | 'PUT' | 'DELETE' | 'HEAD',
|
||||
url: URL,
|
||||
body?: Uint8Array
|
||||
body?: Uint8Array,
|
||||
contentType?: string
|
||||
): Promise<Response> {
|
||||
const payloadHashHex = await sha256Hex(body || new Uint8Array());
|
||||
const amzDate = new Date().toISOString().replace(/[:-]|\.\d{3}/g, '');
|
||||
@@ -407,7 +439,7 @@ async function signedE3Request(
|
||||
'x-amz-content-sha256': payloadHashHex,
|
||||
'x-amz-date': amzDate,
|
||||
};
|
||||
if (method === 'PUT') headers['content-type'] = 'application/zip';
|
||||
if (method === 'PUT') headers['content-type'] = contentType || 'application/octet-stream';
|
||||
|
||||
const authorization = await buildAwsV4Authorization(
|
||||
method,
|
||||
@@ -431,18 +463,26 @@ async function signedE3Request(
|
||||
});
|
||||
}
|
||||
|
||||
async function uploadToE3(config: E3BackupDestination, archive: Uint8Array, fileName: string): Promise<BackupUploadResult> {
|
||||
const objectKey = normalizeE3ObjectKey(config, fileName);
|
||||
async function putToE3(
|
||||
config: E3BackupDestination,
|
||||
relativePath: string,
|
||||
bytes: Uint8Array,
|
||||
options: RemoteBackupFilePutOptions = {}
|
||||
): Promise<void> {
|
||||
const objectKey = normalizeE3ObjectKey(config, relativePath);
|
||||
const url = new URL(`${e3BucketBaseUrl(config).toString()}/${encodePathSegments(objectKey)}`);
|
||||
const response = await signedE3Request(config, 'PUT', url, archive);
|
||||
const response = await signedE3Request(config, 'PUT', url, bytes, options.contentType);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`E3 upload failed: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadToE3(config: E3BackupDestination, archive: Uint8Array, fileName: string): Promise<BackupUploadResult> {
|
||||
await putToE3(config, fileName, archive, { contentType: 'application/zip' });
|
||||
return {
|
||||
provider: 'e3',
|
||||
remotePath: objectKey,
|
||||
remotePath: normalizeE3ObjectKey(config, fileName),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -546,13 +586,26 @@ async function deleteFromE3(config: E3BackupDestination, relativePath: string):
|
||||
}
|
||||
}
|
||||
|
||||
async function existsInE3(config: E3BackupDestination, relativePath: string): Promise<boolean> {
|
||||
const objectKey = normalizeE3ObjectKey(config, relativePath);
|
||||
const url = new URL(`${e3BucketBaseUrl(config).toString()}/${encodePathSegments(objectKey)}`);
|
||||
const response = await signedE3Request(config, 'HEAD', url);
|
||||
if (response.status === 404) return false;
|
||||
if (!response.ok) {
|
||||
throw new Error(`E3 existence check failed: ${response.status}`);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
interface ConfiguredDestinationAdapter {
|
||||
provider: 'webdav' | 'e3';
|
||||
config: WebDavBackupDestination | E3BackupDestination;
|
||||
upload: (config: WebDavBackupDestination | E3BackupDestination, archive: Uint8Array, fileName: string) => Promise<BackupUploadResult>;
|
||||
putFile: (config: WebDavBackupDestination | E3BackupDestination, relativePath: string, bytes: Uint8Array, options?: RemoteBackupFilePutOptions) => Promise<void>;
|
||||
list: (config: WebDavBackupDestination | E3BackupDestination, relativePath: string) => Promise<RemoteBackupListResult>;
|
||||
download: (config: WebDavBackupDestination | E3BackupDestination, relativePath: string) => Promise<RemoteBackupFile>;
|
||||
deleteFile: (config: WebDavBackupDestination | E3BackupDestination, relativePath: string) => Promise<void>;
|
||||
exists: (config: WebDavBackupDestination | E3BackupDestination, relativePath: string) => Promise<boolean>;
|
||||
}
|
||||
|
||||
function resolveConfiguredDestinationAdapter(
|
||||
@@ -565,9 +618,11 @@ function resolveConfiguredDestinationAdapter(
|
||||
provider: 'webdav',
|
||||
config: destination.destination as WebDavBackupDestination,
|
||||
upload: (config, archive, fileName) => uploadToWebDav(config as WebDavBackupDestination, archive, fileName),
|
||||
putFile: (config, relativePath, bytes, options) => putToWebDav(config as WebDavBackupDestination, relativePath, bytes, options),
|
||||
list: (config, relativePath) => listWebDavEntries(config as WebDavBackupDestination, relativePath),
|
||||
download: (config, relativePath) => downloadFromWebDav(config as WebDavBackupDestination, relativePath),
|
||||
deleteFile: (config, relativePath) => deleteFromWebDav(config as WebDavBackupDestination, relativePath),
|
||||
exists: (config, relativePath) => existsInWebDav(config as WebDavBackupDestination, relativePath),
|
||||
};
|
||||
}
|
||||
if (destination.type === 'e3') {
|
||||
@@ -575,9 +630,11 @@ function resolveConfiguredDestinationAdapter(
|
||||
provider: 'e3',
|
||||
config: destination.destination as E3BackupDestination,
|
||||
upload: (config, archive, fileName) => uploadToE3(config as E3BackupDestination, archive, fileName),
|
||||
putFile: (config, relativePath, bytes, options) => putToE3(config as E3BackupDestination, relativePath, bytes, options),
|
||||
list: (config, relativePath) => listE3Entries(config as E3BackupDestination, relativePath),
|
||||
download: (config, relativePath) => downloadFromE3(config as E3BackupDestination, relativePath),
|
||||
deleteFile: (config, relativePath) => deleteFromE3(config as E3BackupDestination, relativePath),
|
||||
exists: (config, relativePath) => existsInE3(config as E3BackupDestination, relativePath),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -609,6 +666,23 @@ export async function deleteRemoteBackupFile(destination: BackupDestinationRecor
|
||||
await adapter.deleteFile(adapter.config, normalized);
|
||||
}
|
||||
|
||||
export async function remoteBackupFileExists(destination: BackupDestinationRecord, relativePath: string): Promise<boolean> {
|
||||
const normalized = normalizeRelativePath(relativePath);
|
||||
const adapter = resolveConfiguredDestinationAdapter(destination);
|
||||
return adapter.exists(adapter.config, normalized);
|
||||
}
|
||||
|
||||
export async function uploadRemoteBackupFile(
|
||||
destination: BackupDestinationRecord,
|
||||
relativePath: string,
|
||||
bytes: Uint8Array,
|
||||
options: RemoteBackupFilePutOptions = {}
|
||||
): Promise<void> {
|
||||
const normalized = normalizeRelativePath(relativePath);
|
||||
const adapter = resolveConfiguredDestinationAdapter(destination);
|
||||
await adapter.putFile(adapter.config, normalized, bytes, options);
|
||||
}
|
||||
|
||||
function compareBackupItemsByRecency(a: RemoteBackupItem, b: RemoteBackupItem, preferredFileName?: string): number {
|
||||
if (preferredFileName) {
|
||||
const aPreferred = a.name === preferredFileName ? 1 : 0;
|
||||
|
||||
Reference in New Issue
Block a user