diff --git a/backend/src/routes/backup.js b/backend/src/routes/backup.js index 51cfaa6..df1f4d4 100644 --- a/backend/src/routes/backup.js +++ b/backend/src/routes/backup.js @@ -1,4 +1,6 @@ import { Router } from 'express'; +import { finished } from 'node:stream/promises'; +import { setTimeout as delay } from 'node:timers/promises'; import { config } from '../config.js'; import { spawnCommand, runRestic } from '../executor.js'; import { appendJobLog, createJob, finishJob, setJobRunning, setJobStep } from '../jobs.js'; @@ -35,17 +37,26 @@ async function runBackupJob(job) { await new Promise((resolve) => setTimeout(resolve, 2000)); setJobStep(job, 'Streaming block device to Restic'); - await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName], { - env: { ...process.env, ...config.resticEnv }, - input: await deviceReadStream(snapshotDevice), - log: (line) => appendJobLog(job, line), - }); + const snapshotStream = await deviceReadStream(snapshotDevice); + try { + await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName], { + env: { ...process.env, ...config.resticEnv }, + input: snapshotStream, + log: (line) => appendJobLog(job, line), + }); + } finally { + if (!snapshotStream.destroyed) { + snapshotStream.destroy(); + } + await finished(snapshotStream, { cleanup: true }).catch(() => {}); + } setJobStep(job, 'Hiding ZFS snapshot device'); await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { log: (line) => appendJobLog(job, line) }); + await settleUdev(job); setJobStep(job, 'Deleting temporary Incus snapshot'); - await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) }); + await deleteIncusSnapshotWithRetry(job, snapshotName); setJobStep(job, 'Applying Restic retention policy'); await runRestic(['forget', '--tag', job.vmName, '--keep-daily', '7', '--prune'], { @@ -71,8 +82,41 @@ async function cleanupBackup(job, zvol, snapshotName) { ignoreExitCode: true, log: (line) => appendJobLog(job, line), }).catch((error) => appendJobLog(job, error.message)); - await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], { + await settleUdev(job); + await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true }); +} + +async function settleUdev(job) { + await spawnCommand('udevadm', ['trigger'], { + ignoreExitCode: true, + log: (line) => appendJobLog(job, line), + }).catch((error) => appendJobLog(job, error.message)); + await spawnCommand('udevadm', ['settle'], { ignoreExitCode: true, log: (line) => appendJobLog(job, line), }).catch((error) => appendJobLog(job, error.message)); } + +async function deleteIncusSnapshotWithRetry(job, snapshotName, options = {}) { + const maxAttempts = 5; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + return await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], { + ignoreExitCode: options.ignoreExitCode, + log: (line) => appendJobLog(job, line), + }); + } catch (error) { + const retryable = error.message.includes('dataset is busy'); + if (!retryable || attempt === maxAttempts) { + if (options.ignoreExitCode) { + appendJobLog(job, error.message); + return null; + } + throw error; + } + appendJobLog(job, `Snapshot device still busy; retrying delete (${attempt}/${maxAttempts}).`); + await delay(2000); + } + } + return null; +}