mirror of
https://github.com/shuaiplus/nodewarden.git
synced 2026-06-20 13:00:39 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5ed7c949c1 | |||
| af70cab766 | |||
| bfea5d0a1c |
@@ -0,0 +1,17 @@
|
||||
# CodeGraph data files
|
||||
# These are local to each machine and should not be committed
|
||||
|
||||
# Database
|
||||
*.db
|
||||
*.db-wal
|
||||
*.db-shm
|
||||
|
||||
# Cache
|
||||
cache/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Hook markers
|
||||
.dirty
|
||||
*.pid
|
||||
@@ -0,0 +1,465 @@
|
||||
import type { Env } from '../types';
|
||||
import type { BackupDestinationRecord } from '../services/backup-config';
|
||||
import {
|
||||
BACKUP_SCHEDULER_WINDOW_MINUTES,
|
||||
requireBackupDestination,
|
||||
hasBackupSlotBetween,
|
||||
isBackupDueNow,
|
||||
loadBackupSettings,
|
||||
} from '../services/backup-config';
|
||||
import {
|
||||
createRemoteBackupTransferSession,
|
||||
downloadRemoteBackupFile,
|
||||
ensureRemoteRestoreCandidate,
|
||||
} from '../services/backup-uploader';
|
||||
import { getBlobObject } from '../services/blob-store';
|
||||
import { StorageService } from '../services/storage';
|
||||
import { notifyUserBackupProgress, notifyUserBackupRestoreProgress } from './notifications-hub';
|
||||
import {
|
||||
executeConfiguredBackup,
|
||||
importAndAuditRemoteBackupFile,
|
||||
} from '../handlers/backup';
|
||||
import { verifyBackupArchiveFileNameChecksum } from '../services/backup-archive';
|
||||
import { zipSync } from 'fflate';
|
||||
|
||||
const BACKUP_JOB_STATE_KEY = 'backup.job.state.v1';
|
||||
const BACKUP_JOB_LEASE_MS = 10 * 60 * 1000;
|
||||
const BACKUP_JOB_HEARTBEAT_MS = 30 * 1000;
|
||||
|
||||
interface BackupJobState {
|
||||
token: string;
|
||||
reason: string;
|
||||
acquiredAt: string;
|
||||
touchedAt: string;
|
||||
expiresAtMs: number;
|
||||
}
|
||||
|
||||
interface RemoteAttachmentChunkRequest {
|
||||
destination: BackupDestinationRecord;
|
||||
attachments: Array<{
|
||||
blobName: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface RemoteAttachmentDownloadRequest {
|
||||
destination: BackupDestinationRecord;
|
||||
blobName?: string | null;
|
||||
}
|
||||
|
||||
interface RemoteAttachmentBatchDownloadRequest {
|
||||
destination: BackupDestinationRecord;
|
||||
blobNames?: string[] | null;
|
||||
}
|
||||
|
||||
interface ConfiguredBackupRunRequest {
|
||||
actorUserId?: string | null;
|
||||
auditMetadata?: Record<string, unknown> | null;
|
||||
destinationId?: string | null;
|
||||
targetDeviceIdentifier?: string | null;
|
||||
trigger?: 'manual' | 'scheduled';
|
||||
}
|
||||
|
||||
interface RemoteBackupRestoreRequest {
|
||||
actorUserId?: string | null;
|
||||
allowChecksumMismatch?: boolean;
|
||||
auditMetadata?: Record<string, unknown> | null;
|
||||
destinationId?: string | null;
|
||||
path?: string | null;
|
||||
replaceExisting?: boolean;
|
||||
targetDeviceIdentifier?: string | null;
|
||||
}
|
||||
|
||||
function badRequest(message: string, status: number = 400): Response {
|
||||
return new Response(JSON.stringify({ error: message }), {
|
||||
status,
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export class BackupTransferRunner {
|
||||
private lastHeartbeatAt = 0;
|
||||
|
||||
constructor(
|
||||
private readonly state: DurableObjectState,
|
||||
private readonly env: Env
|
||||
) {
|
||||
}
|
||||
|
||||
private async acquireJob(reason: string): Promise<string | null> {
|
||||
const nowMs = Date.now();
|
||||
const current = await this.state.storage.get<BackupJobState>(BACKUP_JOB_STATE_KEY);
|
||||
if (current?.expiresAtMs && current.expiresAtMs > nowMs) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const token = crypto.randomUUID();
|
||||
const nowIso = new Date(nowMs).toISOString();
|
||||
await this.state.storage.put<BackupJobState>(BACKUP_JOB_STATE_KEY, {
|
||||
token,
|
||||
reason,
|
||||
acquiredAt: nowIso,
|
||||
touchedAt: nowIso,
|
||||
expiresAtMs: nowMs + BACKUP_JOB_LEASE_MS,
|
||||
});
|
||||
this.lastHeartbeatAt = 0;
|
||||
return token;
|
||||
}
|
||||
|
||||
private async touchJob(token: string): Promise<void> {
|
||||
const nowMs = Date.now();
|
||||
if (nowMs - this.lastHeartbeatAt < BACKUP_JOB_HEARTBEAT_MS) return;
|
||||
this.lastHeartbeatAt = nowMs;
|
||||
|
||||
const current = await this.state.storage.get<BackupJobState>(BACKUP_JOB_STATE_KEY);
|
||||
if (current?.token !== token) return;
|
||||
|
||||
await this.state.storage.put<BackupJobState>(BACKUP_JOB_STATE_KEY, {
|
||||
...current,
|
||||
touchedAt: new Date(nowMs).toISOString(),
|
||||
expiresAtMs: nowMs + BACKUP_JOB_LEASE_MS,
|
||||
});
|
||||
}
|
||||
|
||||
private async releaseJob(token: string): Promise<void> {
|
||||
const current = await this.state.storage.get<BackupJobState>(BACKUP_JOB_STATE_KEY);
|
||||
if (current?.token === token) {
|
||||
await this.state.storage.delete(BACKUP_JOB_STATE_KEY);
|
||||
}
|
||||
}
|
||||
|
||||
private async runConfiguredBackup(request: Request): Promise<Response> {
|
||||
let body: ConfiguredBackupRunRequest;
|
||||
try {
|
||||
body = await request.json<ConfiguredBackupRunRequest>();
|
||||
} catch {
|
||||
return badRequest('Backup run payload is invalid');
|
||||
}
|
||||
|
||||
const trigger = body.trigger === 'scheduled' ? 'scheduled' : 'manual';
|
||||
const actorUserId = String(body.actorUserId || '').trim() || null;
|
||||
if (trigger === 'manual' && !actorUserId) {
|
||||
return badRequest('Manual backup run requires an actor');
|
||||
}
|
||||
|
||||
const token = await this.acquireJob(`${trigger}:${actorUserId || 'system'}`);
|
||||
if (!token) {
|
||||
return badRequest('Another backup run is already in progress', 409);
|
||||
}
|
||||
|
||||
try {
|
||||
await this.touchJob(token);
|
||||
const storage = new StorageService(this.env.DB);
|
||||
const progress = actorUserId
|
||||
? async (event: {
|
||||
operation: 'backup-remote-run';
|
||||
step: string;
|
||||
fileName: string;
|
||||
stageTitle: string;
|
||||
stageDetail: string;
|
||||
done?: boolean;
|
||||
ok?: boolean;
|
||||
error?: string | null;
|
||||
}) => {
|
||||
await notifyUserBackupProgress(
|
||||
this.env,
|
||||
actorUserId,
|
||||
event,
|
||||
String(body.targetDeviceIdentifier || '').trim() || null
|
||||
);
|
||||
}
|
||||
: null;
|
||||
|
||||
const result = await executeConfiguredBackup(
|
||||
this.env,
|
||||
storage,
|
||||
actorUserId,
|
||||
trigger,
|
||||
body.destinationId || null,
|
||||
() => this.touchJob(token),
|
||||
progress,
|
||||
body.auditMetadata || null
|
||||
);
|
||||
const settings = await loadBackupSettings(storage, this.env, 'UTC');
|
||||
|
||||
return new Response(JSON.stringify({
|
||||
object: 'backup-runner-result',
|
||||
result,
|
||||
settings,
|
||||
}), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
return badRequest(error instanceof Error ? error.message : 'Backup run failed', 500);
|
||||
} finally {
|
||||
await this.releaseJob(token);
|
||||
}
|
||||
}
|
||||
|
||||
private async runScheduledBackups(): Promise<Response> {
|
||||
const token = await this.acquireJob('scheduled');
|
||||
if (!token) {
|
||||
return badRequest('Another backup run is already in progress', 409);
|
||||
}
|
||||
|
||||
let completed = 0;
|
||||
try {
|
||||
await this.touchJob(token);
|
||||
const storage = new StorageService(this.env.DB);
|
||||
let scanStartMs = Date.now();
|
||||
|
||||
while (true) {
|
||||
await this.touchJob(token);
|
||||
const settings = await loadBackupSettings(storage, this.env, 'UTC');
|
||||
const now = new Date();
|
||||
const dueDestinations = settings.destinations.filter((destination) =>
|
||||
isBackupDueNow(destination, now, BACKUP_SCHEDULER_WINDOW_MINUTES)
|
||||
|| hasBackupSlotBetween(destination, new Date(scanStartMs), now)
|
||||
);
|
||||
|
||||
if (!dueDestinations.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
scanStartMs = now.getTime();
|
||||
for (const destination of dueDestinations) {
|
||||
await this.touchJob(token);
|
||||
await executeConfiguredBackup(
|
||||
this.env,
|
||||
storage,
|
||||
null,
|
||||
'scheduled',
|
||||
destination.id,
|
||||
() => this.touchJob(token)
|
||||
);
|
||||
completed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return new Response(JSON.stringify({
|
||||
ok: true,
|
||||
completed,
|
||||
}), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
return badRequest(error instanceof Error ? error.message : 'Scheduled backup failed', 500);
|
||||
} finally {
|
||||
await this.releaseJob(token);
|
||||
}
|
||||
}
|
||||
|
||||
private async restoreRemoteBackup(request: Request): Promise<Response> {
|
||||
let body: RemoteBackupRestoreRequest;
|
||||
try {
|
||||
body = await request.json<RemoteBackupRestoreRequest>();
|
||||
} catch {
|
||||
return badRequest('Remote restore payload is invalid');
|
||||
}
|
||||
|
||||
const actorUserId = String(body.actorUserId || '').trim() || null;
|
||||
if (!actorUserId) {
|
||||
return badRequest('Remote restore requires an actor');
|
||||
}
|
||||
|
||||
const token = await this.acquireJob(`restore:${actorUserId}`);
|
||||
if (!token) {
|
||||
return badRequest('Another backup or restore run is already in progress', 409);
|
||||
}
|
||||
|
||||
try {
|
||||
await this.touchJob(token);
|
||||
const storage = new StorageService(this.env.DB);
|
||||
const settings = await loadBackupSettings(storage, this.env, 'UTC');
|
||||
const destination = requireBackupDestination(settings, body.destinationId || null);
|
||||
const path = ensureRemoteRestoreCandidate(String(body.path || ''));
|
||||
const restoreFileNameFromPath = path.split('/').pop() || path;
|
||||
const targetDeviceIdentifier = String(body.targetDeviceIdentifier || '').trim() || null;
|
||||
const replaceExisting = !!body.replaceExisting;
|
||||
|
||||
await notifyUserBackupRestoreProgress(
|
||||
this.env,
|
||||
actorUserId,
|
||||
{
|
||||
operation: 'backup-restore',
|
||||
source: 'remote',
|
||||
step: 'remote_fetch_archive',
|
||||
fileName: restoreFileNameFromPath,
|
||||
stageTitle: 'txt_backup_restore_progress_remote_fetch_title',
|
||||
stageDetail: 'txt_backup_restore_progress_remote_fetch_detail',
|
||||
replaceExisting,
|
||||
},
|
||||
targetDeviceIdentifier
|
||||
);
|
||||
|
||||
const remoteFile = await downloadRemoteBackupFile(destination, path);
|
||||
const checksumOk = await verifyBackupArchiveFileNameChecksum(remoteFile.bytes, remoteFile.fileName || path);
|
||||
if (!checksumOk && !body.allowChecksumMismatch) {
|
||||
return badRequest('Remote backup file checksum does not match its filename');
|
||||
}
|
||||
|
||||
const result = await importAndAuditRemoteBackupFile(
|
||||
this.env,
|
||||
storage,
|
||||
actorUserId,
|
||||
remoteFile,
|
||||
destination,
|
||||
path,
|
||||
replaceExisting,
|
||||
!checksumOk,
|
||||
body.auditMetadata || null,
|
||||
targetDeviceIdentifier
|
||||
);
|
||||
|
||||
return new Response(JSON.stringify(result.result), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
return badRequest(error instanceof Error ? error.message : 'Remote backup restore failed', 500);
|
||||
} finally {
|
||||
await this.releaseJob(token);
|
||||
}
|
||||
}
|
||||
|
||||
async fetch(request: Request): Promise<Response> {
|
||||
const url = new URL(request.url);
|
||||
if (request.method !== 'POST') {
|
||||
return badRequest('Not found', 404);
|
||||
}
|
||||
|
||||
if (url.pathname === '/internal/run-configured-backup') {
|
||||
return this.runConfiguredBackup(request);
|
||||
}
|
||||
|
||||
if (url.pathname === '/internal/run-scheduled-backups') {
|
||||
return this.runScheduledBackups();
|
||||
}
|
||||
|
||||
if (url.pathname === '/internal/restore-remote-backup') {
|
||||
return this.restoreRemoteBackup(request);
|
||||
}
|
||||
|
||||
if (url.pathname === '/internal/download-remote-attachment') {
|
||||
let body: RemoteAttachmentDownloadRequest;
|
||||
try {
|
||||
body = await request.json<RemoteAttachmentDownloadRequest>();
|
||||
} catch {
|
||||
return badRequest('Remote attachment download payload is invalid');
|
||||
}
|
||||
const blobName = String(body?.blobName || '').trim();
|
||||
if (!body?.destination || !blobName) {
|
||||
return badRequest('Remote attachment download payload is invalid');
|
||||
}
|
||||
const file = await downloadRemoteBackupFile(body.destination, `attachments/${blobName}`).catch(() => null);
|
||||
if (!file) {
|
||||
return badRequest('Remote attachment not found', 404);
|
||||
}
|
||||
return new Response(file.bytes, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': file.contentType || 'application/octet-stream',
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (url.pathname === '/internal/download-remote-attachment-batch') {
|
||||
let body: RemoteAttachmentBatchDownloadRequest;
|
||||
try {
|
||||
body = await request.json<RemoteAttachmentBatchDownloadRequest>();
|
||||
} catch {
|
||||
return badRequest('Remote attachment batch download payload is invalid');
|
||||
}
|
||||
const blobNames = Array.from(new Set(
|
||||
(Array.isArray(body?.blobNames) ? body.blobNames : [])
|
||||
.map((blobName) => String(blobName || '').trim())
|
||||
.filter(Boolean)
|
||||
));
|
||||
if (!body?.destination || !blobNames.length || blobNames.length > 40) {
|
||||
return badRequest('Remote attachment batch download payload is invalid');
|
||||
}
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const entries: Array<{ blobName: string; path: string }> = [];
|
||||
const files: Record<string, Uint8Array> = {};
|
||||
for (let i = 0; i < blobNames.length; i += 1) {
|
||||
const blobName = blobNames[i];
|
||||
const file = await downloadRemoteBackupFile(body.destination, `attachments/${blobName}`).catch(() => null);
|
||||
if (!file) continue;
|
||||
const path = `files/${i}.bin`;
|
||||
entries.push({ blobName, path });
|
||||
files[path] = file.bytes;
|
||||
}
|
||||
files['manifest.json'] = encoder.encode(JSON.stringify({ version: 1, entries }));
|
||||
|
||||
return new Response(zipSync(files), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/zip',
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (url.pathname !== '/internal/upload-attachment-chunk') {
|
||||
return badRequest('Not found', 404);
|
||||
}
|
||||
|
||||
let body: RemoteAttachmentChunkRequest;
|
||||
try {
|
||||
body = await request.json<RemoteAttachmentChunkRequest>();
|
||||
} catch {
|
||||
return badRequest('Attachment chunk payload is invalid');
|
||||
}
|
||||
|
||||
if (!body?.destination || !Array.isArray(body.attachments)) {
|
||||
return badRequest('Attachment chunk payload is invalid');
|
||||
}
|
||||
|
||||
const remoteSession = createRemoteBackupTransferSession(body.destination);
|
||||
let uploaded = 0;
|
||||
|
||||
for (const attachment of body.attachments) {
|
||||
const blobName = String(attachment?.blobName || '').trim();
|
||||
if (!blobName) {
|
||||
return badRequest('Attachment chunk payload is invalid');
|
||||
}
|
||||
|
||||
const object = await getBlobObject(this.env, blobName);
|
||||
if (!object) {
|
||||
return badRequest(`Attachment blob missing for ${blobName}`, 409);
|
||||
}
|
||||
|
||||
const bytes = new Uint8Array(await new Response(object.body).arrayBuffer());
|
||||
await remoteSession.putFile(`attachments/${blobName}`, bytes, {
|
||||
contentType: object.contentType,
|
||||
});
|
||||
uploaded += 1;
|
||||
}
|
||||
|
||||
return new Response(JSON.stringify({
|
||||
ok: true,
|
||||
uploaded,
|
||||
}), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
+379
-235
@@ -1,21 +1,20 @@
|
||||
import type { Env, User } from '../types';
|
||||
import { errorResponse, jsonResponse } from '../utils/response';
|
||||
import { generateUUID } from '../utils/uuid';
|
||||
import {
|
||||
type BackupArchiveBundle,
|
||||
buildBackupArchive,
|
||||
inspectBackupArchiveFileNameChecksum,
|
||||
parseBackupArchive,
|
||||
verifyBackupArchiveFileNameChecksum,
|
||||
} from '../services/backup-archive';
|
||||
import {
|
||||
type BackupDestinationRecord,
|
||||
type BackupSettingsInput,
|
||||
BACKUP_SCHEDULER_WINDOW_MINUTES,
|
||||
type BackupSettings,
|
||||
type WebDavBackupDestination,
|
||||
getBackupLocalDateKey,
|
||||
getDefaultBackupSettings,
|
||||
getBackupSettingsRepairState,
|
||||
hasBackupSlotBetween,
|
||||
isBackupDueNow,
|
||||
loadBackupSettings,
|
||||
normalizeBackupSettingsInput,
|
||||
normalizeImportedBackupSettings,
|
||||
@@ -31,6 +30,7 @@ import {
|
||||
} from '../services/backup-import';
|
||||
import {
|
||||
type RemoteBackupTransferSession,
|
||||
type RemoteBackupFile,
|
||||
createRemoteBackupTransferSession,
|
||||
deleteRemoteBackupFile,
|
||||
downloadRemoteBackupFile,
|
||||
@@ -43,6 +43,7 @@ import { StorageService } from '../services/storage';
|
||||
import { auditRequestMetadata, writeAuditEvent } from '../services/audit-events';
|
||||
import { getBlobObject } from '../services/blob-store';
|
||||
import { notifyUserBackupProgress, notifyUserBackupRestoreProgress } from '../durable/notifications-hub';
|
||||
import { unzipSync } from 'fflate';
|
||||
|
||||
function isAdmin(user: User): boolean {
|
||||
return user.role === 'admin' && user.status === 'active';
|
||||
@@ -86,102 +87,6 @@ function getBackupDestinationSummary(destination: BackupDestinationRecord | null
|
||||
};
|
||||
}
|
||||
|
||||
const BACKUP_RUNNER_LOCK_KEY = 'backup.runner.lock.v1';
|
||||
const BACKUP_RUNNER_LEASE_MS = 10 * 60 * 1000;
|
||||
const BACKUP_RUNNER_HEARTBEAT_MS = 30 * 1000;
|
||||
|
||||
// CONTRACT:
|
||||
// The runner lock is a config-row lease, not a queue. It only prevents two
|
||||
// backup/restore jobs from overlapping. Manual runs return conflict when the
|
||||
// lease is held; scheduled runs skip quietly. Never export this row in backups.
|
||||
interface BackupRunnerLease {
|
||||
token: string;
|
||||
touch: () => Promise<void>;
|
||||
release: () => Promise<void>;
|
||||
}
|
||||
|
||||
async function acquireBackupRunnerLease(env: Env, reason: string): Promise<BackupRunnerLease | null> {
|
||||
const token = generateUUID();
|
||||
const nowMs = Date.now();
|
||||
const expiresAtMs = nowMs + BACKUP_RUNNER_LEASE_MS;
|
||||
const value = JSON.stringify({
|
||||
token,
|
||||
reason,
|
||||
acquiredAt: new Date(nowMs).toISOString(),
|
||||
touchedAt: new Date(nowMs).toISOString(),
|
||||
expiresAtMs,
|
||||
});
|
||||
const result = await env.DB
|
||||
.prepare(
|
||||
`INSERT INTO config(key, value) VALUES(?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
||||
WHERE COALESCE(CAST(json_extract(config.value, '$.expiresAtMs') AS INTEGER), 0) <= ?`
|
||||
)
|
||||
.bind(BACKUP_RUNNER_LOCK_KEY, value, nowMs)
|
||||
.run();
|
||||
|
||||
if ((result.meta?.changes || 0) < 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
token,
|
||||
touch: async () => {
|
||||
const nextNowMs = Date.now();
|
||||
const nextValue = JSON.stringify({
|
||||
token,
|
||||
reason,
|
||||
acquiredAt: new Date(nowMs).toISOString(),
|
||||
touchedAt: new Date(nextNowMs).toISOString(),
|
||||
expiresAtMs: nextNowMs + BACKUP_RUNNER_LEASE_MS,
|
||||
});
|
||||
await env.DB
|
||||
.prepare(
|
||||
`UPDATE config
|
||||
SET value = ?
|
||||
WHERE key = ?
|
||||
AND json_extract(value, '$.token') = ?`
|
||||
)
|
||||
.bind(nextValue, BACKUP_RUNNER_LOCK_KEY, token)
|
||||
.run();
|
||||
},
|
||||
release: async () => {
|
||||
await env.DB
|
||||
.prepare(
|
||||
`DELETE FROM config
|
||||
WHERE key = ?
|
||||
AND json_extract(value, '$.token') = ?`
|
||||
)
|
||||
.bind(BACKUP_RUNNER_LOCK_KEY, token)
|
||||
.run();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function withBackupRunnerLease<T>(
|
||||
env: Env,
|
||||
reason: string,
|
||||
task: (keepAlive: () => Promise<void>) => Promise<T>
|
||||
): Promise<T | null> {
|
||||
const lease = await acquireBackupRunnerLease(env, reason);
|
||||
if (!lease) return null;
|
||||
|
||||
let lastHeartbeatAt = 0;
|
||||
const keepAlive = async () => {
|
||||
const nowMs = Date.now();
|
||||
if (nowMs - lastHeartbeatAt < BACKUP_RUNNER_HEARTBEAT_MS) return;
|
||||
lastHeartbeatAt = nowMs;
|
||||
await lease.touch();
|
||||
};
|
||||
|
||||
try {
|
||||
await keepAlive();
|
||||
return await task(keepAlive);
|
||||
} finally {
|
||||
await lease.release();
|
||||
}
|
||||
}
|
||||
|
||||
function ensureBackupBlobName(value: string): string {
|
||||
const normalized = String(value || '').trim().replace(/\\/g, '/').replace(/^\/+|\/+$/g, '');
|
||||
if (!normalized) {
|
||||
@@ -201,6 +106,37 @@ interface RemoteAttachmentIndexPayload {
|
||||
blobs: Record<string, { sizeBytes: number; updatedAt: string }>;
|
||||
}
|
||||
|
||||
const REMOTE_ATTACHMENT_SYNC_EXTERNAL_SUBREQUEST_LIMIT = 50;
|
||||
const REMOTE_ATTACHMENT_SYNC_SUBREQUEST_RESERVE = 6;
|
||||
const REMOTE_ATTACHMENT_SYNC_MAX_WEB_DAV_BATCH_SIZE = 18;
|
||||
const REMOTE_ATTACHMENT_SYNC_MAX_S3_BATCH_SIZE = 40;
|
||||
const REMOTE_ATTACHMENT_RESTORE_BATCH_SIZE = 40;
|
||||
|
||||
function countRemotePathSegments(value: string): number {
|
||||
return String(value || '').replace(/\\/g, '/').split('/').filter(Boolean).length;
|
||||
}
|
||||
|
||||
function getRemoteAttachmentSyncBatchSize(destination: BackupDestinationRecord): number {
|
||||
if (destination.type === 's3') {
|
||||
return REMOTE_ATTACHMENT_SYNC_MAX_S3_BATCH_SIZE;
|
||||
}
|
||||
|
||||
const remotePath = String((destination.destination as WebDavBackupDestination).remotePath || '');
|
||||
const fixedWebDavDirectoryCalls = countRemotePathSegments(remotePath) + 1; // remotePath plus the shared "attachments" dir.
|
||||
const available = REMOTE_ATTACHMENT_SYNC_EXTERNAL_SUBREQUEST_LIMIT
|
||||
- REMOTE_ATTACHMENT_SYNC_SUBREQUEST_RESERVE
|
||||
- fixedWebDavDirectoryCalls;
|
||||
|
||||
if (available < 2) {
|
||||
throw new Error('WebDAV remote backup path is too deep for safe attachment batching');
|
||||
}
|
||||
|
||||
return Math.max(1, Math.min(
|
||||
REMOTE_ATTACHMENT_SYNC_MAX_WEB_DAV_BATCH_SIZE,
|
||||
Math.floor(available / 2)
|
||||
));
|
||||
}
|
||||
|
||||
async function loadRemoteAttachmentIndex(session: RemoteBackupTransferSession): Promise<Map<string, number>> {
|
||||
try {
|
||||
const file = await session.download(REMOTE_ATTACHMENT_INDEX_PATH);
|
||||
@@ -256,7 +192,39 @@ async function saveRemoteAttachmentIndex(
|
||||
});
|
||||
}
|
||||
|
||||
async function executeConfiguredBackup(
|
||||
async function uploadRemoteAttachmentChunk(
|
||||
env: Env,
|
||||
destination: BackupDestinationRecord,
|
||||
attachments: Array<{ blobName: string }>
|
||||
): Promise<void> {
|
||||
if (!attachments.length) return;
|
||||
const id = env.BACKUP_TRANSFER_RUNNER.idFromName('remote-attachment-sync');
|
||||
const stub = env.BACKUP_TRANSFER_RUNNER.get(id);
|
||||
const response = await stub.fetch('https://backup-transfer/internal/upload-attachment-chunk', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
destination,
|
||||
attachments,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
let message = `Attachment sync failed: ${response.status}`;
|
||||
try {
|
||||
const payload = await response.json<{ error?: string }>();
|
||||
if (payload?.error) {
|
||||
message = payload.error;
|
||||
}
|
||||
} catch {
|
||||
// Ignore JSON parse failures and preserve the status-based error.
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
export async function executeConfiguredBackup(
|
||||
env: Env,
|
||||
storage: StorageService,
|
||||
actorUserId: string | null,
|
||||
@@ -331,25 +299,20 @@ async function executeConfiguredBackup(
|
||||
if (destination.includeAttachments) {
|
||||
await touchLease();
|
||||
const remoteAttachmentIndex = await loadRemoteAttachmentIndex(remoteSession);
|
||||
let attachmentIndexChanged = false;
|
||||
for (const attachment of archive.manifest.attachmentBlobs || []) {
|
||||
const pendingAttachments = (archive.manifest.attachmentBlobs || [])
|
||||
.filter((attachment) => remoteAttachmentIndex.get(attachment.blobName) !== attachment.sizeBytes);
|
||||
const attachmentSyncBatchSize = getRemoteAttachmentSyncBatchSize(destination);
|
||||
for (let i = 0; i < pendingAttachments.length; i += attachmentSyncBatchSize) {
|
||||
await touchLease();
|
||||
if (remoteAttachmentIndex.get(attachment.blobName) === attachment.sizeBytes) {
|
||||
continue;
|
||||
}
|
||||
const remotePath = `attachments/${attachment.blobName}`;
|
||||
const object = await getBlobObject(env, attachment.blobName);
|
||||
if (!object) {
|
||||
throw new Error(`Attachment blob missing for ${attachment.blobName}`);
|
||||
}
|
||||
const bytes = new Uint8Array(await new Response(object.body).arrayBuffer());
|
||||
await remoteSession.putFile(remotePath, bytes, {
|
||||
contentType: object.contentType,
|
||||
});
|
||||
remoteAttachmentIndex.set(attachment.blobName, attachment.sizeBytes);
|
||||
attachmentIndexChanged = true;
|
||||
const chunk = pendingAttachments
|
||||
.slice(i, i + attachmentSyncBatchSize)
|
||||
.map((attachment) => ({ blobName: attachment.blobName }));
|
||||
await uploadRemoteAttachmentChunk(env, destination, chunk);
|
||||
}
|
||||
if (attachmentIndexChanged) {
|
||||
if (pendingAttachments.length) {
|
||||
for (const attachment of pendingAttachments) {
|
||||
remoteAttachmentIndex.set(attachment.blobName, attachment.sizeBytes);
|
||||
}
|
||||
await touchLease();
|
||||
await saveRemoteAttachmentIndex(remoteSession, remoteAttachmentIndex);
|
||||
}
|
||||
@@ -474,14 +437,293 @@ async function executeConfiguredBackup(
|
||||
}
|
||||
}
|
||||
|
||||
interface DurableBackupRunResponse {
|
||||
result: {
|
||||
fileName: string;
|
||||
fileSize: number;
|
||||
remotePath: string;
|
||||
provider: string;
|
||||
};
|
||||
settings: BackupSettings;
|
||||
}
|
||||
|
||||
async function runConfiguredBackupInDurableObject(
|
||||
env: Env,
|
||||
payload: {
|
||||
actorUserId: string | null;
|
||||
auditMetadata?: Record<string, unknown> | null;
|
||||
destinationId?: string | null;
|
||||
targetDeviceIdentifier?: string | null;
|
||||
trigger: 'manual' | 'scheduled';
|
||||
}
|
||||
): Promise<DurableBackupRunResponse | null> {
|
||||
const id = env.BACKUP_TRANSFER_RUNNER.idFromName('configured-backup-runner');
|
||||
const stub = env.BACKUP_TRANSFER_RUNNER.get(id);
|
||||
const response = await stub.fetch('https://backup-transfer/internal/run-configured-backup', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
if (response.status === 409) {
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) {
|
||||
let message = `Backup run failed: ${response.status}`;
|
||||
try {
|
||||
const body = await response.json<{ error?: string }>();
|
||||
if (body?.error) message = body.error;
|
||||
} catch {
|
||||
// Preserve the status-based message when the DO returns a non-JSON error.
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
const body = await response.json<DurableBackupRunResponse>();
|
||||
if (!body?.result || !body?.settings) {
|
||||
throw new Error('Backup run response is invalid');
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
async function runScheduledBackupsInDurableObject(env: Env): Promise<void> {
|
||||
const id = env.BACKUP_TRANSFER_RUNNER.idFromName('configured-backup-runner');
|
||||
const stub = env.BACKUP_TRANSFER_RUNNER.get(id);
|
||||
const response = await stub.fetch('https://backup-transfer/internal/run-scheduled-backups', {
|
||||
method: 'POST',
|
||||
});
|
||||
if (response.status === 409) {
|
||||
return;
|
||||
}
|
||||
if (!response.ok) {
|
||||
let message = `Scheduled backup failed: ${response.status}`;
|
||||
try {
|
||||
const body = await response.json<{ error?: string }>();
|
||||
if (body?.error) message = body.error;
|
||||
} catch {
|
||||
// Preserve the status-based message when the DO returns a non-JSON error.
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadRemoteAttachmentViaDurableObject(
|
||||
env: Env,
|
||||
destination: BackupDestinationRecord,
|
||||
blobName: string
|
||||
): Promise<Uint8Array | null> {
|
||||
const id = env.BACKUP_TRANSFER_RUNNER.idFromName('remote-attachment-restore');
|
||||
const stub = env.BACKUP_TRANSFER_RUNNER.get(id);
|
||||
const response = await stub.fetch('https://backup-transfer/internal/download-remote-attachment', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
destination,
|
||||
blobName,
|
||||
}),
|
||||
});
|
||||
if (response.status === 404) {
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new Error(`Remote attachment download failed: ${response.status}`);
|
||||
}
|
||||
return new Uint8Array(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
async function downloadRemoteAttachmentBatchViaDurableObject(
|
||||
env: Env,
|
||||
destination: BackupDestinationRecord,
|
||||
blobNames: string[]
|
||||
): Promise<Map<string, Uint8Array>> {
|
||||
const names = Array.from(new Set(blobNames.map((blobName) => String(blobName || '').trim()).filter(Boolean)));
|
||||
const result = new Map<string, Uint8Array>();
|
||||
if (!names.length) return result;
|
||||
|
||||
const id = env.BACKUP_TRANSFER_RUNNER.idFromName('remote-attachment-restore');
|
||||
const stub = env.BACKUP_TRANSFER_RUNNER.get(id);
|
||||
const response = await stub.fetch('https://backup-transfer/internal/download-remote-attachment-batch', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
destination,
|
||||
blobNames: names,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Remote attachment batch download failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const files = unzipSync(new Uint8Array(await response.arrayBuffer()));
|
||||
const manifestBytes = files['manifest.json'];
|
||||
if (!manifestBytes) return result;
|
||||
const manifest = JSON.parse(new TextDecoder().decode(manifestBytes)) as {
|
||||
entries?: Array<{ blobName?: string; path?: string }>;
|
||||
};
|
||||
for (const entry of manifest.entries || []) {
|
||||
const blobName = String(entry.blobName || '').trim();
|
||||
const path = String(entry.path || '').trim();
|
||||
const bytes = path ? files[path] : null;
|
||||
if (blobName && bytes) {
|
||||
result.set(blobName, bytes);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function collectExternalRemoteAttachmentBlobNames(archiveBytes: Uint8Array): string[] {
|
||||
const parsed = parseBackupArchive(archiveBytes, { allowExternalAttachmentBlobs: true });
|
||||
const refs = new Map(
|
||||
(parsed.payload.manifest.attachmentBlobs || [])
|
||||
.map((item) => [`${String(item.cipherId || '').trim()}/${String(item.attachmentId || '').trim()}`, item])
|
||||
);
|
||||
const names: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const row of parsed.payload.db.attachments || []) {
|
||||
const cipherId = String(row.cipher_id || '').trim();
|
||||
const attachmentId = String(row.id || '').trim();
|
||||
const inlinePath = `attachments/${cipherId}/${attachmentId}.bin`;
|
||||
if (parsed.files[inlinePath]) continue;
|
||||
const ref = refs.get(`${cipherId}/${attachmentId}`);
|
||||
const blobName = String(ref?.blobName || '').trim();
|
||||
if (blobName && !seen.has(blobName)) {
|
||||
seen.add(blobName);
|
||||
names.push(blobName);
|
||||
}
|
||||
}
|
||||
|
||||
return names;
|
||||
}
|
||||
|
||||
function toImportStatusCode(message: string): number {
|
||||
const lower = message.toLowerCase();
|
||||
if (lower.includes('checksum')) return 400;
|
||||
if (lower.includes('invalid backup') || lower.includes('invalid json')) return 400;
|
||||
if (lower.includes('fresh instance')) return 409;
|
||||
if (lower.includes('not configured') || lower.includes('kv')) return 409;
|
||||
return 500;
|
||||
}
|
||||
|
||||
export async function importAndAuditRemoteBackupFile(
|
||||
env: Env,
|
||||
storage: StorageService,
|
||||
actorUserId: string,
|
||||
remoteFile: RemoteBackupFile,
|
||||
destination: BackupDestinationRecord,
|
||||
remotePath: string,
|
||||
replaceExisting: boolean,
|
||||
checksumMismatchAccepted: boolean,
|
||||
auditMetadata: Record<string, unknown> | null = null,
|
||||
targetDeviceIdentifier: string | null = null
|
||||
): Promise<BackupImportExecutionResult> {
|
||||
const restoreFileName = remoteFile.fileName || remotePath.split('/').pop() || remotePath;
|
||||
const externalAttachmentBlobNames = collectExternalRemoteAttachmentBlobNames(remoteFile.bytes);
|
||||
const externalAttachmentCache = new Map<string, Uint8Array | null>();
|
||||
const progress: BackupRestoreProgressReporter = async (event) => {
|
||||
await notifyUserBackupRestoreProgress(
|
||||
env,
|
||||
actorUserId,
|
||||
{
|
||||
operation: 'backup-restore',
|
||||
...event,
|
||||
},
|
||||
targetDeviceIdentifier
|
||||
);
|
||||
};
|
||||
const result = await importRemoteBackupArchiveBytes(
|
||||
remoteFile.bytes,
|
||||
env,
|
||||
actorUserId,
|
||||
replaceExisting,
|
||||
{
|
||||
loadAttachment: async (blobName) => {
|
||||
const normalized = String(blobName || '').trim();
|
||||
if (!normalized) return null;
|
||||
if (externalAttachmentCache.has(normalized)) {
|
||||
return externalAttachmentCache.get(normalized) || null;
|
||||
}
|
||||
|
||||
const start = Math.max(0, externalAttachmentBlobNames.indexOf(normalized));
|
||||
const batchNames = externalAttachmentBlobNames
|
||||
.slice(start, start + REMOTE_ATTACHMENT_RESTORE_BATCH_SIZE)
|
||||
.filter((name) => !externalAttachmentCache.has(name));
|
||||
if (!batchNames.includes(normalized)) {
|
||||
batchNames.unshift(normalized);
|
||||
}
|
||||
|
||||
try {
|
||||
const batch = await downloadRemoteAttachmentBatchViaDurableObject(env, destination, batchNames);
|
||||
for (const name of batchNames) {
|
||||
externalAttachmentCache.set(name, batch.get(name) || null);
|
||||
}
|
||||
} catch {
|
||||
externalAttachmentCache.set(normalized, await downloadRemoteAttachmentViaDurableObject(env, destination, normalized).catch(() => null));
|
||||
}
|
||||
return externalAttachmentCache.get(normalized) || null;
|
||||
},
|
||||
},
|
||||
progress,
|
||||
restoreFileName
|
||||
);
|
||||
await writeAuditLog(storage, result.auditActorUserId, 'admin.backup.import', 'backup', null, {
|
||||
users: result.result.imported.users,
|
||||
ciphers: result.result.imported.ciphers,
|
||||
attachments: result.result.imported.attachmentFiles,
|
||||
skippedAttachments: result.result.skipped.attachments,
|
||||
skippedReason: result.result.skipped.reason,
|
||||
replaceExisting,
|
||||
...getBackupDestinationSummary(destination),
|
||||
remotePath,
|
||||
bytes: remoteFile.bytes.byteLength,
|
||||
trigger: 'remote',
|
||||
checksumMismatchAccepted,
|
||||
...(auditMetadata || {}),
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
async function restoreRemoteBackupInDurableObject(
|
||||
env: Env,
|
||||
payload: {
|
||||
actorUserId: string;
|
||||
allowChecksumMismatch?: boolean;
|
||||
auditMetadata?: Record<string, unknown> | null;
|
||||
destinationId?: string | null;
|
||||
path: string;
|
||||
replaceExisting?: boolean;
|
||||
targetDeviceIdentifier?: string | null;
|
||||
}
|
||||
): Promise<BackupImportExecutionResult['result'] | null> {
|
||||
const id = env.BACKUP_TRANSFER_RUNNER.idFromName('configured-backup-runner');
|
||||
const stub = env.BACKUP_TRANSFER_RUNNER.get(id);
|
||||
const response = await stub.fetch('https://backup-transfer/internal/restore-remote-backup', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
if (response.status === 409) {
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) {
|
||||
let message = `Remote backup restore failed: ${response.status}`;
|
||||
try {
|
||||
const body = await response.json<{ error?: string }>();
|
||||
if (body?.error) message = body.error;
|
||||
} catch {
|
||||
// Preserve the status-based message when the DO returns a non-JSON error.
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
return response.json<BackupImportExecutionResult['result']>();
|
||||
}
|
||||
|
||||
async function runImportAndAudit(
|
||||
env: Env,
|
||||
request: Request,
|
||||
@@ -526,30 +768,7 @@ async function runImportAndAudit(
|
||||
}
|
||||
|
||||
export async function runScheduledBackupIfDue(env: Env): Promise<void> {
|
||||
await withBackupRunnerLease(env, 'scheduled', async (keepAlive) => {
|
||||
const storage = new StorageService(env.DB);
|
||||
let scanStartMs = Date.now();
|
||||
|
||||
while (true) {
|
||||
await keepAlive();
|
||||
const settings = await loadBackupSettings(storage, env, 'UTC');
|
||||
const now = new Date();
|
||||
const dueDestinations = settings.destinations.filter((destination) =>
|
||||
isBackupDueNow(destination, now, BACKUP_SCHEDULER_WINDOW_MINUTES)
|
||||
|| hasBackupSlotBetween(destination, new Date(scanStartMs), now)
|
||||
);
|
||||
|
||||
if (!dueDestinations.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
scanStartMs = now.getTime();
|
||||
for (const destination of dueDestinations) {
|
||||
await keepAlive();
|
||||
await executeConfiguredBackup(env, storage, null, 'scheduled', destination.id, keepAlive);
|
||||
}
|
||||
}
|
||||
});
|
||||
await runScheduledBackupsInDurableObject(env);
|
||||
}
|
||||
|
||||
export async function handleGetAdminBackupSettings(request: Request, env: Env, actorUser: User): Promise<Response> {
|
||||
@@ -661,33 +880,12 @@ export async function handleRunAdminConfiguredBackup(request: Request, env: Env,
|
||||
return errorResponse('Backup run payload is invalid', 400);
|
||||
}
|
||||
|
||||
const targetDeviceIdentifier = String(request.headers.get('X-NodeWarden-Acting-Device-Id') || '').trim() || null;
|
||||
const progress = async (event: {
|
||||
operation: 'backup-remote-run';
|
||||
step: string;
|
||||
fileName: string;
|
||||
stageTitle: string;
|
||||
stageDetail: string;
|
||||
done?: boolean;
|
||||
ok?: boolean;
|
||||
error?: string | null;
|
||||
}) => {
|
||||
await notifyUserBackupProgress(env, actorUser.id, event, targetDeviceIdentifier);
|
||||
};
|
||||
const outcome = await withBackupRunnerLease(env, `manual:${actorUser.id}`, async (keepAlive) => {
|
||||
const storage = new StorageService(env.DB);
|
||||
const result = await executeConfiguredBackup(
|
||||
env,
|
||||
storage,
|
||||
actorUser.id,
|
||||
'manual',
|
||||
body?.destinationId || null,
|
||||
keepAlive,
|
||||
progress,
|
||||
auditRequestMetadata(request)
|
||||
);
|
||||
const settings = await loadBackupSettings(storage, env, 'UTC');
|
||||
return { result, settings };
|
||||
const outcome = await runConfiguredBackupInDurableObject(env, {
|
||||
actorUserId: actorUser.id,
|
||||
auditMetadata: auditRequestMetadata(request),
|
||||
destinationId: body?.destinationId || null,
|
||||
targetDeviceIdentifier: String(request.headers.get('X-NodeWarden-Acting-Device-Id') || '').trim() || null,
|
||||
trigger: 'manual',
|
||||
});
|
||||
if (!outcome) {
|
||||
return errorResponse('Another backup run is already in progress', 409);
|
||||
@@ -803,76 +1001,22 @@ export async function handleRestoreAdminRemoteBackup(request: Request, env: Env,
|
||||
return errorResponse('Remote restore payload is invalid', 400);
|
||||
}
|
||||
|
||||
const storage = new StorageService(env.DB);
|
||||
try {
|
||||
const settings = await loadBackupSettings(storage, env, 'UTC');
|
||||
const destination = requireBackupDestination(settings, body.destinationId || null);
|
||||
const path = ensureRemoteRestoreCandidate(String(body.path || ''));
|
||||
const targetDeviceIdentifier = String(request.headers.get('X-NodeWarden-Acting-Device-Id') || '').trim() || null;
|
||||
const restoreFileNameFromPath = path.split('/').pop() || path;
|
||||
await notifyUserBackupRestoreProgress(
|
||||
env,
|
||||
actorUser.id,
|
||||
{
|
||||
operation: 'backup-restore',
|
||||
source: 'remote',
|
||||
step: 'remote_fetch_archive',
|
||||
fileName: restoreFileNameFromPath,
|
||||
stageTitle: 'txt_backup_restore_progress_remote_fetch_title',
|
||||
stageDetail: 'txt_backup_restore_progress_remote_fetch_detail',
|
||||
replaceExisting: !!body.replaceExisting,
|
||||
},
|
||||
targetDeviceIdentifier
|
||||
);
|
||||
const remoteFile = await downloadRemoteBackupFile(destination, path);
|
||||
const checksumOk = await verifyBackupArchiveFileNameChecksum(remoteFile.bytes, remoteFile.fileName || path);
|
||||
if (!checksumOk && !body.allowChecksumMismatch) {
|
||||
return errorResponse('Remote backup file checksum does not match its filename', 400);
|
||||
const imported = await restoreRemoteBackupInDurableObject(env, {
|
||||
actorUserId: actorUser.id,
|
||||
allowChecksumMismatch: !!body.allowChecksumMismatch,
|
||||
auditMetadata: auditRequestMetadata(request),
|
||||
destinationId: body.destinationId || null,
|
||||
path,
|
||||
replaceExisting: !!body.replaceExisting,
|
||||
targetDeviceIdentifier,
|
||||
});
|
||||
if (!imported) {
|
||||
return errorResponse('Another backup or restore run is already in progress', 409);
|
||||
}
|
||||
const restoreFileName = remoteFile.fileName || path.split('/').pop() || path;
|
||||
const progress: BackupRestoreProgressReporter = async (event) => {
|
||||
await notifyUserBackupRestoreProgress(
|
||||
env,
|
||||
actorUser.id,
|
||||
{
|
||||
operation: 'backup-restore',
|
||||
...event,
|
||||
},
|
||||
targetDeviceIdentifier
|
||||
);
|
||||
};
|
||||
const imported = await (async () => {
|
||||
const storage = new StorageService(env.DB);
|
||||
const result = await importRemoteBackupArchiveBytes(
|
||||
remoteFile.bytes,
|
||||
env,
|
||||
actorUser.id,
|
||||
!!body.replaceExisting,
|
||||
{
|
||||
loadAttachment: async (blobName) => {
|
||||
const file = await downloadRemoteBackupFile(destination, `attachments/${blobName}`).catch(() => null);
|
||||
return file?.bytes || null;
|
||||
},
|
||||
},
|
||||
progress,
|
||||
restoreFileName
|
||||
);
|
||||
await writeAuditLog(storage, result.auditActorUserId, 'admin.backup.import', 'backup', null, {
|
||||
users: result.result.imported.users,
|
||||
ciphers: result.result.imported.ciphers,
|
||||
attachments: result.result.imported.attachmentFiles,
|
||||
skippedAttachments: result.result.skipped.attachments,
|
||||
skippedReason: result.result.skipped.reason,
|
||||
replaceExisting: !!body.replaceExisting,
|
||||
...getBackupDestinationSummary(destination),
|
||||
remotePath: path,
|
||||
bytes: remoteFile.bytes.byteLength,
|
||||
trigger: 'remote',
|
||||
checksumMismatchAccepted: !checksumOk,
|
||||
}, request);
|
||||
return result;
|
||||
})();
|
||||
return jsonResponse(imported.result);
|
||||
return jsonResponse(imported);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Remote backup restore failed';
|
||||
return errorResponse(message, toImportStatusCode(message));
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Env } from './types';
|
||||
import { NotificationsHub } from './durable/notifications-hub';
|
||||
import { BackupTransferRunner } from './durable/backup-transfer-runner';
|
||||
import { handleRequest } from './router';
|
||||
import { StorageService } from './services/storage';
|
||||
import { applyCors, jsonResponse } from './utils/response';
|
||||
@@ -127,3 +128,4 @@ export default {
|
||||
};
|
||||
|
||||
export { NotificationsHub };
|
||||
export { BackupTransferRunner };
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
export interface Env {
|
||||
DB: D1Database;
|
||||
NOTIFICATIONS_HUB: DurableObjectNamespace;
|
||||
BACKUP_TRANSFER_RUNNER: DurableObjectNamespace;
|
||||
ASSETS?: {
|
||||
fetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response>;
|
||||
};
|
||||
|
||||
@@ -91,6 +91,7 @@ const COMMON_IMPORT_SOURCE_IDS: ImportSourceId[] = [
|
||||
'lastpass',
|
||||
'dashlane_csv',
|
||||
'dashlane_json',
|
||||
'keepass_csv',
|
||||
'keepass_xml',
|
||||
'keepassx_csv',
|
||||
];
|
||||
|
||||
@@ -23,6 +23,7 @@ export const IMPORT_SOURCES = [
|
||||
{ id: 'lastpass', label: 'LastPass (csv)' },
|
||||
{ id: 'dashlane_csv', label: 'Dashlane (csv)' },
|
||||
{ id: 'dashlane_json', label: 'Dashlane (json)' },
|
||||
{ id: 'keepass_csv', label: 'KeePass 1.x (csv)' },
|
||||
{ id: 'keepass_xml', label: 'KeePass 2 (xml)' },
|
||||
{ id: 'keepassx_csv', label: 'KeePassX (csv)' },
|
||||
{ id: 'arc_csv', label: 'Arc (csv)' },
|
||||
|
||||
@@ -198,6 +198,7 @@ export function parseEncryptrCsv(textRaw: string): CiphersImportPayload {
|
||||
export function parseKeePassXCsv(textRaw: string): CiphersImportPayload {
|
||||
const rows = parseCsv(textRaw);
|
||||
const result: CiphersImportPayload = { ciphers: [], folders: [], folderRelationships: [] };
|
||||
const standardColumns = new Set(['Group', 'Title', 'Username', 'Password', 'URL', 'Notes', 'TOTP']);
|
||||
for (const row of rows) {
|
||||
if (!txt(row.Title)) continue;
|
||||
const cipher = makeLoginCipher();
|
||||
@@ -209,12 +210,34 @@ export function parseKeePassXCsv(textRaw: string): CiphersImportPayload {
|
||||
login.totp = val(row.TOTP);
|
||||
const uri = normalizeUri(row.URL || '');
|
||||
login.uris = uri ? [{ uri, match: null }] : null;
|
||||
for (const [key, value] of Object.entries(row)) {
|
||||
if (standardColumns.has(key)) continue;
|
||||
processKvp(cipher, key, value, false);
|
||||
}
|
||||
const idx = result.ciphers.push(cipher) - 1;
|
||||
addFolder(result, txt(row.Group).replace(/^Root\//, ''), idx);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function parseKeePassCsv(textRaw: string): CiphersImportPayload {
|
||||
const rows = parseCsv(textRaw);
|
||||
const result: CiphersImportPayload = { ciphers: [], folders: [], folderRelationships: [] };
|
||||
for (const row of rows) {
|
||||
if (!txt(row.Account)) continue;
|
||||
const cipher = makeLoginCipher();
|
||||
cipher.name = val(row.Account, '--');
|
||||
cipher.notes = val(row.Comments);
|
||||
const login = cipher.login as Record<string, unknown>;
|
||||
login.username = val(row['Login Name']);
|
||||
login.password = val(row.Password);
|
||||
const uri = normalizeUri(row['Web Site'] || '');
|
||||
login.uris = uri ? [{ uri, match: null }] : null;
|
||||
result.ciphers.push(cipher);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function parseLastPassCsv(textRaw: string): CiphersImportPayload {
|
||||
const rows = parseCsv(textRaw);
|
||||
const result: CiphersImportPayload = { ciphers: [], folders: [], folderRelationships: [] };
|
||||
@@ -350,7 +373,8 @@ export function parseKeePassXml(textRaw: string): CiphersImportPayload {
|
||||
const cipher = makeLoginCipher();
|
||||
for (const s of qd(entry, 'String')) {
|
||||
const key = txt(qd(s, 'Key')[0]?.textContent);
|
||||
const value = txt(qd(s, 'Value')[0]?.textContent);
|
||||
const valueNode = qd(s, 'Value')[0];
|
||||
const value = txt(valueNode?.textContent);
|
||||
if (!value) continue;
|
||||
const login = cipher.login as Record<string, unknown>;
|
||||
if (key === 'Title') cipher.name = value;
|
||||
@@ -361,6 +385,11 @@ export function parseKeePassXml(textRaw: string): CiphersImportPayload {
|
||||
login.uris = uri ? [{ uri, match: null }] : null;
|
||||
} else if (key === 'otp') login.totp = value.replace('key=', '');
|
||||
else if (key === 'Notes') cipher.notes = `${txt(cipher.notes)}${txt(cipher.notes) ? '\n' : ''}${value}`;
|
||||
else {
|
||||
const hidden = ['True', 'true', '1'].includes(valueNode?.getAttribute('ProtectInMemory') || '')
|
||||
|| ['True', 'true', '1'].includes(valueNode?.getAttribute('Protected') || '');
|
||||
processKvp(cipher, key, value, hidden);
|
||||
}
|
||||
}
|
||||
const idx = result.ciphers.push(cipher) - 1;
|
||||
if (!isRoot && folder >= 0) result.folderRelationships.push({ key: idx, value: folder });
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
parseDashlaneCsv,
|
||||
parseDashlaneJson,
|
||||
parseEncryptrCsv,
|
||||
parseKeePassCsv,
|
||||
parseKeePassXCsv,
|
||||
parseKeePassXml,
|
||||
parseLastPassCsv,
|
||||
@@ -75,6 +76,7 @@ const IMPORT_SOURCE_PARSERS: Record<ImportSourceId, (textRaw: string) => Ciphers
|
||||
lastpass: parseLastPassCsv,
|
||||
dashlane_csv: parseDashlaneCsv,
|
||||
dashlane_json: parseDashlaneJson,
|
||||
keepass_csv: parseKeePassCsv,
|
||||
keepass_xml: parseKeePassXml,
|
||||
keepassx_csv: parseKeePassXCsv,
|
||||
arc_csv: parseArcCsv,
|
||||
|
||||
@@ -22,9 +22,17 @@ database_name = "nodewarden-db"
|
||||
name = "NOTIFICATIONS_HUB"
|
||||
class_name = "NotificationsHub"
|
||||
|
||||
[[durable_objects.bindings]]
|
||||
name = "BACKUP_TRANSFER_RUNNER"
|
||||
class_name = "BackupTransferRunner"
|
||||
|
||||
[[kv_namespaces]]
|
||||
binding = "ATTACHMENTS_KV"
|
||||
|
||||
[[migrations]]
|
||||
tag = "v1-notifications-hub"
|
||||
new_sqlite_classes = [ "NotificationsHub" ]
|
||||
|
||||
[[migrations]]
|
||||
tag = "v2-backup-transfer-runner"
|
||||
new_sqlite_classes = [ "BackupTransferRunner" ]
|
||||
|
||||
@@ -22,6 +22,10 @@ database_name = "nodewarden-db"
|
||||
name = "NOTIFICATIONS_HUB"
|
||||
class_name = "NotificationsHub"
|
||||
|
||||
[[durable_objects.bindings]]
|
||||
name = "BACKUP_TRANSFER_RUNNER"
|
||||
class_name = "BackupTransferRunner"
|
||||
|
||||
[[r2_buckets]]
|
||||
binding = "ATTACHMENTS"
|
||||
bucket_name = "nodewarden-attachments"
|
||||
@@ -29,3 +33,7 @@ bucket_name = "nodewarden-attachments"
|
||||
[[migrations]]
|
||||
tag = "v1-notifications-hub"
|
||||
new_sqlite_classes = [ "NotificationsHub" ]
|
||||
|
||||
[[migrations]]
|
||||
tag = "v2-backup-transfer-runner"
|
||||
new_sqlite_classes = [ "BackupTransferRunner" ]
|
||||
|
||||
Reference in New Issue
Block a user