added backend repair for ZVOL close

backend/src/routes/backup.js

@@ -1,4 +1,6 @@
 import { Router } from 'express';
+import { finished } from 'node:stream/promises';
+import { setTimeout as delay } from 'node:timers/promises';
 import { config } from '../config.js';
 import { spawnCommand, runRestic } from '../executor.js';
 import { appendJobLog, createJob, finishJob, setJobRunning, setJobStep } from '../jobs.js';
@@ -35,17 +37,26 @@ async function runBackupJob(job) {
     await new Promise((resolve) => setTimeout(resolve, 2000));
 
     setJobStep(job, 'Streaming block device to Restic');
-    await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName], {
-      env: { ...process.env, ...config.resticEnv },
-      input: await deviceReadStream(snapshotDevice),
-      log: (line) => appendJobLog(job, line),
-    });
+    const snapshotStream = await deviceReadStream(snapshotDevice);
+    try {
+      await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName], {
+        env: { ...process.env, ...config.resticEnv },
+        input: snapshotStream,
+        log: (line) => appendJobLog(job, line),
+      });
+    } finally {
+      if (!snapshotStream.destroyed) {
+        snapshotStream.destroy();
+      }
+      await finished(snapshotStream, { cleanup: true }).catch(() => {});
+    }
 
     setJobStep(job, 'Hiding ZFS snapshot device');
     await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { log: (line) => appendJobLog(job, line) });
+    await settleUdev(job);
 
     setJobStep(job, 'Deleting temporary Incus snapshot');
-    await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
+    await deleteIncusSnapshotWithRetry(job, snapshotName);
 
     setJobStep(job, 'Applying Restic retention policy');
     await runRestic(['forget', '--tag', job.vmName, '--keep-daily', '7', '--prune'], {
@@ -71,8 +82,41 @@ async function cleanupBackup(job, zvol, snapshotName) {
     ignoreExitCode: true,
     log: (line) => appendJobLog(job, line),
   }).catch((error) => appendJobLog(job, error.message));
-  await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], {
+  await settleUdev(job);
+  await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
+}
+
+async function settleUdev(job) {
+  await spawnCommand('udevadm', ['trigger'], {
+    ignoreExitCode: true,
+    log: (line) => appendJobLog(job, line),
+  }).catch((error) => appendJobLog(job, error.message));
+  await spawnCommand('udevadm', ['settle'], {
     ignoreExitCode: true,
     log: (line) => appendJobLog(job, line),
   }).catch((error) => appendJobLog(job, error.message));
 }
+
+async function deleteIncusSnapshotWithRetry(job, snapshotName, options = {}) {
+  const maxAttempts = 5;
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      return await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], {
+        ignoreExitCode: options.ignoreExitCode,
+        log: (line) => appendJobLog(job, line),
+      });
+    } catch (error) {
+      const retryable = error.message.includes('dataset is busy');
+      if (!retryable || attempt === maxAttempts) {
+        if (options.ignoreExitCode) {
+          appendJobLog(job, error.message);
+          return null;
+        }
+        throw error;
+      }
+      appendJobLog(job, `Snapshot device still busy; retrying delete (${attempt}/${maxAttempts}).`);
+      await delay(2000);
+    }
+  }
+  return null;
+}