added install script for agent

changed backend to agent

agent/src/config.js

@@ -0,0 +1,158 @@
+import dotenv from 'dotenv';
+import { readFile, rename, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+
+dotenv.config();
+
+const envPath = path.resolve(process.cwd(), '.env');
+const minApiTokenLength = 32;
+
+export const requiredEnv = [
+  'AWS_ACCESS_KEY_ID',
+  'AWS_SECRET_ACCESS_KEY',
+  'RESTIC_REPOSITORY',
+  'RESTIC_PASSWORD',
+  'ZFS_POOL_NAME',
+];
+
+export const config = {
+  port: Number(process.env.PORT || 3000),
+  host: process.env.HOST || '0.0.0.0',
+  apiToken: process.env.API_TOKEN || '',
+  httpsEnabled: process.env.HTTPS_ENABLED === 'true',
+  tlsCertFile: process.env.TLS_CERT_FILE || '',
+  tlsKeyFile: process.env.TLS_KEY_FILE || '',
+  agentDatabasePath: path.resolve(process.cwd(), process.env.AGENT_DATABASE_PATH || './agent.sqlite'),
+  allowedManagementIps: (process.env.ALLOWED_MANAGEMENT_IPS || '')
+    .split(',')
+    .map((value) => value.trim())
+    .filter(Boolean),
+  zfsPoolName: process.env.ZFS_POOL_NAME || '',
+  resticEnv: {
+    AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID || '',
+    AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY || '',
+    RESTIC_REPOSITORY: process.env.RESTIC_REPOSITORY || '',
+    RESTIC_PASSWORD: process.env.RESTIC_PASSWORD || '',
+  },
+  retention: {
+    keepHourly: Number(process.env.RESTIC_KEEP_HOURLY || 0),
+    keepDaily: Number(process.env.RESTIC_KEEP_DAILY || 7),
+    keepWeekly: Number(process.env.RESTIC_KEEP_WEEKLY || 0),
+    keepMonthly: Number(process.env.RESTIC_KEEP_MONTHLY || 0),
+  },
+};
+
+if (!config.apiToken || config.apiToken.length < minApiTokenLength) {
+  throw new Error(`API_TOKEN is required and must be at least ${minApiTokenLength} characters long.`);
+}
+
+if (config.httpsEnabled && (!config.tlsCertFile || !config.tlsKeyFile)) {
+  throw new Error('TLS_CERT_FILE and TLS_KEY_FILE are required when HTTPS_ENABLED=true.');
+}
+
+export const editableEnv = [
+  { key: 'AWS_ACCESS_KEY_ID', label: 'AWS access key ID', required: true, secret: true },
+  { key: 'AWS_SECRET_ACCESS_KEY', label: 'AWS secret access key', required: true, secret: true },
+  { key: 'RESTIC_REPOSITORY', label: 'Restic repository', required: true, secret: false },
+  { key: 'RESTIC_PASSWORD', label: 'Restic password', required: true, secret: true },
+  { key: 'ZFS_POOL_NAME', label: 'ZFS pool name', required: true, secret: false },
+  { key: 'RESTIC_KEEP_HOURLY', label: 'Keep hourly snapshots', required: false, secret: false },
+  { key: 'RESTIC_KEEP_DAILY', label: 'Keep daily snapshots', required: false, secret: false },
+  { key: 'RESTIC_KEEP_WEEKLY', label: 'Keep weekly snapshots', required: false, secret: false },
+  { key: 'RESTIC_KEEP_MONTHLY', label: 'Keep monthly snapshots', required: false, secret: false },
+  { key: 'PORT', label: 'API port', required: false, secret: false },
+  { key: 'HOST', label: 'API bind host', required: false, secret: false },
+  { key: 'API_TOKEN', label: 'API token', required: true, secret: true },
+  { key: 'HTTPS_ENABLED', label: 'Enable HTTPS', required: false, secret: false },
+  { key: 'TLS_CERT_FILE', label: 'TLS certificate file', required: false, secret: false },
+  { key: 'TLS_KEY_FILE', label: 'TLS private key file', required: false, secret: false },
+  { key: 'AGENT_DATABASE_PATH', label: 'Agent database path', required: false, secret: false },
+  { key: 'ALLOWED_MANAGEMENT_IPS', label: 'Allowed management IPs', required: false, secret: false },
+];
+
+export function missingEnvVars() {
+  return requiredEnv.filter((key) => !process.env[key]);
+}
+
+export function resticProcessEnv() {
+  return {
+    ...process.env,
+    ...config.resticEnv,
+  };
+}
+
+export async function readEnvSettings() {
+  const fileValues = await readEnvFile();
+  return editableEnv.map((field) => ({
+    ...field,
+    value: fileValues[field.key] ?? process.env[field.key] ?? '',
+  }));
+}
+
+export async function writeEnvSettings(values) {
+  const allowedKeys = new Set(editableEnv.map((field) => field.key));
+  const currentValues = await readEnvFile();
+  const nextValues = { ...currentValues };
+
+  for (const [key, value] of Object.entries(values || {})) {
+    if (!allowedKeys.has(key)) continue;
+    if (key === 'API_TOKEN' && String(value || '').length < minApiTokenLength) {
+      const error = new Error(`API_TOKEN must be at least ${minApiTokenLength} characters long.`);
+      error.status = 400;
+      throw error;
+    }
+    nextValues[key] = String(value ?? '');
+  }
+
+  const body = editableEnv
+    .map(({ key }) => `${key}=${quoteEnvValue(nextValues[key] || '')}`)
+    .join('\n');
+
+  const tempPath = `${envPath}.tmp`;
+  await writeFile(tempPath, `${body}\n`, { mode: 0o600 });
+  await rename(tempPath, envPath);
+
+  applyRuntimeEnv(nextValues);
+  return readEnvSettings();
+}
+
+async function readEnvFile() {
+  try {
+    const content = await readFile(envPath, 'utf8');
+    return dotenv.parse(content);
+  } catch (error) {
+    if (error.code === 'ENOENT') return {};
+    throw error;
+  }
+}
+
+function quoteEnvValue(value) {
+  const escaped = String(value).replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+  return `"${escaped}"`;
+}
+
+function applyRuntimeEnv(values) {
+  for (const [key, value] of Object.entries(values)) {
+    process.env[key] = value;
+  }
+  config.port = Number(process.env.PORT || 3000);
+  config.host = process.env.HOST || '0.0.0.0';
+  config.apiToken = process.env.API_TOKEN || '';
+  config.httpsEnabled = process.env.HTTPS_ENABLED === 'true';
+  config.tlsCertFile = process.env.TLS_CERT_FILE || '';
+  config.tlsKeyFile = process.env.TLS_KEY_FILE || '';
+  config.agentDatabasePath = path.resolve(process.cwd(), process.env.AGENT_DATABASE_PATH || './agent.sqlite');
+  config.allowedManagementIps = (process.env.ALLOWED_MANAGEMENT_IPS || '')
+    .split(',')
+    .map((value) => value.trim())
+    .filter(Boolean);
+  config.zfsPoolName = process.env.ZFS_POOL_NAME || '';
+  config.resticEnv.AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID || '';
+  config.resticEnv.AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY || '';
+  config.resticEnv.RESTIC_REPOSITORY = process.env.RESTIC_REPOSITORY || '';
+  config.resticEnv.RESTIC_PASSWORD = process.env.RESTIC_PASSWORD || '';
+  config.retention.keepHourly = Number(process.env.RESTIC_KEEP_HOURLY || 0);
+  config.retention.keepDaily = Number(process.env.RESTIC_KEEP_DAILY || 7);
+  config.retention.keepWeekly = Number(process.env.RESTIC_KEEP_WEEKLY || 0);
+  config.retention.keepMonthly = Number(process.env.RESTIC_KEEP_MONTHLY || 0);
+}

agent/src/executor.js

@@ -0,0 +1,161 @@
+import { spawn } from 'node:child_process';
+import { PassThrough, Transform } from 'node:stream';
+import { resticProcessEnv } from './config.js';
+
+export class CommandError extends Error {
+  constructor(message, result) {
+    super(message);
+    this.name = 'CommandError';
+    this.result = result;
+  }
+}
+
+export function spawnCommand(command, args = [], options = {}) {
+  const {
+    env = process.env,
+    input = null,
+    log = null,
+    ignoreExitCode = false,
+    cwd = process.cwd(),
+  } = options;
+
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, {
+      cwd,
+      env,
+      shell: false,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+
+    let stdout = '';
+    let stderr = '';
+
+    child.stdout.on('data', (chunk) => {
+      const text = chunk.toString();
+      stdout += text;
+      log?.(text.trimEnd());
+    });
+
+    child.stderr.on('data', (chunk) => {
+      const text = chunk.toString();
+      stderr += text;
+      log?.(text.trimEnd());
+    });
+
+    child.on('error', (error) => {
+      reject(error);
+    });
+
+    child.on('close', (exitCode) => {
+      const result = { stdout, stderr, exitCode };
+      if (exitCode !== 0 && !ignoreExitCode) {
+        reject(new CommandError(stderr.trim() || `${command} exited with ${exitCode}`, result));
+        return;
+      }
+      resolve(result);
+    });
+
+    if (input) {
+      input.on('error', (error) => {
+        child.kill('SIGTERM');
+        reject(error);
+      });
+      input.pipe(child.stdin);
+    } else {
+      child.stdin.end();
+    }
+  });
+}
+
+export function runRestic(args, options = {}) {
+  return spawnCommand('restic', args, {
+    ...options,
+    env: resticProcessEnv(),
+  });
+}
+
+export async function resticSnapshotFileSize(snapshotId, filename) {
+  const result = await runRestic(['ls', '--json', snapshotId]);
+  const wanted = `/${filename}`;
+  for (const line of result.stdout.split('\n')) {
+    if (!line.trim()) continue;
+    const entry = JSON.parse(line);
+    if (entry.type === 'file' && (entry.path === filename || entry.path === wanted || entry.path?.endsWith(wanted))) {
+      return Number(entry.size);
+    }
+  }
+  const error = new Error(`Could not verify Restic file size for ${filename} in snapshot ${snapshotId}.`);
+  error.status = 500;
+  throw error;
+}
+
+export function createProgressStream(totalBytes, onProgress) {
+  let currentBytes = 0;
+  return new Transform({
+    transform(chunk, _encoding, callback) {
+      currentBytes += chunk.length;
+      onProgress?.({
+        currentBytes,
+        totalBytes,
+        percent: totalBytes ? (currentBytes / totalBytes) * 100 : 0,
+      });
+      callback(null, chunk);
+    },
+  });
+}
+
+export function streamResticDumpToDd(snapshotId, filename, outputPath, log, progress = null) {
+  return new Promise((resolve, reject) => {
+    const restic = spawn('restic', ['dump', snapshotId, filename], {
+      env: resticProcessEnv(),
+      shell: false,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    const dd = spawn('dd', [`of=${outputPath}`, 'bs=4M', 'conv=sparse', 'status=none'], {
+      env: process.env,
+      shell: false,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+
+    const errors = [];
+    let resticClosed = false;
+    let ddClosed = false;
+    const pipe = new PassThrough();
+    let restoreStream = restic.stdout.pipe(pipe);
+    if (progress) {
+      restoreStream = restoreStream.pipe(createProgressStream(progress.totalBytes, progress.onProgress));
+    }
+    restoreStream.pipe(dd.stdin);
+
+    restic.stderr.on('data', (chunk) => log?.(chunk.toString().trimEnd()));
+    dd.stderr.on('data', (chunk) => log?.(chunk.toString().trimEnd()));
+    restic.on('error', reject);
+    dd.on('error', reject);
+
+    restic.on('close', (code) => {
+      resticClosed = true;
+      if (code !== 0) {
+        errors.push(`restic dump exited with ${code}`);
+        dd.stdin.destroy();
+      }
+      maybeFinish();
+    });
+
+    dd.on('close', (code) => {
+      ddClosed = true;
+      if (code !== 0) {
+        errors.push(`dd exited with ${code}`);
+      }
+      maybeFinish();
+    });
+
+    function maybeFinish() {
+      if (!resticClosed || !ddClosed) return;
+      if (errors.length) {
+        reject(new Error(errors.join('; ')));
+        return;
+      }
+      resolve();
+    }
+  });
+}

agent/src/index.js

@@ -0,0 +1,74 @@
+import cors from 'cors';
+import express from 'express';
+import http from 'node:http';
+import https from 'node:https';
+import { readFile } from 'node:fs/promises';
+import { config } from './config.js';
+import { backupRouter } from './routes/backup.js';
+import { healthRouter } from './routes/health.js';
+import { jobsRouter } from './routes/jobs.js';
+import { restoreRouter } from './routes/restore.js';
+import { schedulesRouter } from './routes/schedules.js';
+import { settingsRouter } from './routes/settings.js';
+import { snapshotsRouter } from './routes/snapshots.js';
+import { vmsRouter } from './routes/vms.js';
+import { startScheduler } from './scheduler.js';
+
+const app = express();
+
+app.use(cors());
+app.use(express.json());
+
+app.use((req, res, next) => {
+  if (config.allowedManagementIps.length && !config.allowedManagementIps.includes(normalizeIp(req.ip))) {
+    res.status(403).json({ error: 'Forbidden management source.' });
+    return;
+  }
+  const header = req.get('authorization') || '';
+  if (header === `Bearer ${config.apiToken}`) {
+    next();
+    return;
+  }
+  res.status(401).json({ error: 'Unauthorized.' });
+});
+
+function normalizeIp(value) {
+  return String(value || '').replace(/^::ffff:/, '');
+}
+
+app.use('/api/health', healthRouter);
+app.use('/api/vms', vmsRouter);
+app.use('/api/snapshots', snapshotsRouter);
+app.use('/api/jobs', jobsRouter);
+app.use('/api/backup', backupRouter);
+app.use('/api/restore', restoreRouter);
+app.use('/api/schedules', schedulesRouter);
+app.use('/api/settings', settingsRouter);
+
+app.use((error, _req, res, _next) => {
+  const status = error.status || 500;
+  res.status(status).json({ error: error.message || 'Internal server error.' });
+});
+
+startServer().catch((error) => {
+  console.error(`Failed to start Incus backup API: ${error.message}`);
+  process.exit(1);
+});
+
+startScheduler().catch((error) => {
+  console.error(`Failed to start scheduler: ${error.message}`);
+});
+
+async function startServer() {
+  const protocol = config.httpsEnabled ? 'https' : 'http';
+  const server = config.httpsEnabled
+    ? https.createServer({
+      cert: await readFile(config.tlsCertFile),
+      key: await readFile(config.tlsKeyFile),
+    }, app)
+    : http.createServer(app);
+
+  server.listen(config.port, config.host, () => {
+    console.log(`Incus backup API listening on ${protocol}://${config.host}:${config.port}`);
+  });
+}

agent/src/jobs.js

@@ -0,0 +1,187 @@
+import crypto from 'node:crypto';
+import { mkdirSync } from 'node:fs';
+import { DatabaseSync } from 'node:sqlite';
+import path from 'node:path';
+import { config } from './config.js';
+
+const jobs = new Map();
+const locks = new Map();
+const maxJobs = 100;
+
+mkdirSync(path.dirname(config.agentDatabasePath), { recursive: true, mode: 0o700 });
+const db = new DatabaseSync(config.agentDatabasePath);
+
+migrate();
+loadPersistedJobs();
+
+export function createJob(type, vmName) {
+  if (locks.has(vmName)) {
+    const activeJob = jobs.get(locks.get(vmName));
+    const error = new Error(`A ${activeJob?.type || 'job'} job is already active for ${vmName}.`);
+    error.status = 409;
+    throw error;
+  }
+
+  const id = `job_${crypto.randomBytes(8).toString('hex')}`;
+  const now = new Date().toISOString();
+  const job = {
+    id,
+    type,
+    vmName,
+    status: 'queued',
+    startedAt: now,
+    finishedAt: null,
+    currentStep: 'Queued',
+    progress: {
+      percent: 0,
+      detail: 'Queued',
+      currentBytes: 0,
+      totalBytes: null,
+    },
+    logs: [],
+    error: null,
+  };
+
+  jobs.set(id, job);
+  locks.set(vmName, id);
+  persistJob(job);
+  trimJobs();
+  return job;
+}
+
+export function getJob(id) {
+  return jobs.get(id) || null;
+}
+
+export function listJobs() {
+  return [...jobs.values()].sort((a, b) => b.startedAt.localeCompare(a.startedAt));
+}
+
+export function activeJobForVm(vmName) {
+  const id = locks.get(vmName);
+  return id ? getJob(id) : null;
+}
+
+export function latestJobForVm(vmName) {
+  return listJobs().find((job) => job.vmName === vmName) || null;
+}
+
+export function setJobRunning(job, step) {
+  job.status = 'running';
+  setJobStep(job, step);
+}
+
+export function setJobStep(job, step) {
+  job.currentStep = step;
+  job.progress = {
+    ...job.progress,
+    detail: step,
+  };
+  appendJobLog(job, `==> ${step}`);
+  persistJob(job);
+}
+
+export function setJobProgress(job, progress = {}) {
+  const nextPercent = progress.percent ?? job.progress?.percent ?? 0;
+  job.progress = {
+    ...job.progress,
+    ...progress,
+    percent: Math.max(0, Math.min(100, Number(nextPercent) || 0)),
+  };
+  persistJob(job);
+}
+
+export function appendJobLog(job, line) {
+  if (!line) return;
+  job.logs.push(...String(line).split('\n').filter(Boolean));
+  if (job.logs.length > 1000) {
+    job.logs = job.logs.slice(-1000);
+  }
+  persistJob(job);
+}
+
+export function finishJob(job, status, error = null) {
+  job.status = status;
+  job.finishedAt = new Date().toISOString();
+  job.currentStep = status === 'success' ? 'Completed' : 'Failed';
+  job.progress = {
+    ...job.progress,
+    percent: status === 'success' ? 100 : job.progress?.percent || 0,
+    detail: status === 'success' ? 'Completed' : 'Failed',
+  };
+  job.error = error ? String(error.message || error) : null;
+  locks.delete(job.vmName);
+  persistJob(job);
+}
+
+function trimJobs() {
+  const allJobs = listJobs();
+  for (const job of allJobs.slice(maxJobs)) {
+    if (job.status !== 'running' && job.status !== 'queued') {
+      jobs.delete(job.id);
+      db.prepare('DELETE FROM jobs WHERE id = ?').run(job.id);
+    }
+  }
+}
+
+function migrate() {
+  db.exec(`
+    PRAGMA journal_mode = WAL;
+    CREATE TABLE IF NOT EXISTS jobs (
+      id TEXT PRIMARY KEY,
+      vm_name TEXT NOT NULL,
+      type TEXT NOT NULL,
+      status TEXT NOT NULL,
+      started_at TEXT NOT NULL,
+      finished_at TEXT,
+      updated_at TEXT NOT NULL,
+      payload TEXT NOT NULL
+    );
+  `);
+}
+
+function loadPersistedJobs() {
+  const now = new Date().toISOString();
+  const rows = db.prepare('SELECT payload FROM jobs ORDER BY started_at DESC LIMIT ?').all(maxJobs);
+  for (const row of rows.reverse()) {
+    const job = JSON.parse(row.payload);
+    if (['queued', 'running'].includes(job.status)) {
+      job.status = 'failed';
+      job.finishedAt = now;
+      job.currentStep = 'Failed';
+      job.progress = {
+        ...job.progress,
+        detail: 'Failed',
+      };
+      job.error = 'Agent restarted while this job was active.';
+      job.logs = [...(job.logs || []), 'Agent restarted while this job was active; job marked failed.'].slice(-1000);
+    }
+    jobs.set(job.id, job);
+    persistJob(job);
+  }
+  trimJobs();
+}
+
+function persistJob(job) {
+  db.prepare(`
+    INSERT INTO jobs (id, vm_name, type, status, started_at, finished_at, updated_at, payload)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      vm_name = excluded.vm_name,
+      type = excluded.type,
+      status = excluded.status,
+      started_at = excluded.started_at,
+      finished_at = excluded.finished_at,
+      updated_at = excluded.updated_at,
+      payload = excluded.payload
+  `).run(
+    job.id,
+    job.vmName,
+    job.type,
+    job.status,
+    job.startedAt,
+    job.finishedAt,
+    new Date().toISOString(),
+    JSON.stringify(job),
+  );
+}

agent/src/routes/backup.js

@@ -0,0 +1,358 @@
+import { Router } from 'express';
+import { spawn } from 'node:child_process';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { finished } from 'node:stream/promises';
+import { setTimeout as delay } from 'node:timers/promises';
+import { config } from '../config.js';
+import { createProgressStream, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js';
+import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js';
+import { validateVmExists } from '../validators.js';
+
+export const backupRouter = Router();
+
+backupRouter.post('/:vmName', async (req, res, next) => {
+  try {
+    const vmName = req.params.vmName;
+    const job = await startBackupForVm(vmName);
+    res.status(202).json({ jobId: job.id, message: 'Backup job started.' });
+  } catch (error) {
+    next(error);
+  }
+});
+
+export async function startBackupForVm(vmName) {
+  const instance = await validateVmExists(vmName);
+  const job = createJob('backup', vmName);
+  job.instanceType = instance.type;
+  runBackupJob(job, instance).catch(() => {});
+  return job;
+}
+
+export async function runBackupJob(job, instance = null) {
+  const currentInstance = instance || await validateVmExists(job.vmName);
+  if (currentInstance.type === 'container') {
+    await runContainerBackupJob(job);
+    return;
+  }
+
+  const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
+  const snapshotName = `s3-backup-${timestamp}`;
+  const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
+  const snapshotDevice = `/dev/zvol/${zvol}@snapshot-${snapshotName}`;
+
+  try {
+    const totalBytes = await zfsVolumeSize(zvol);
+
+    setJobRunning(job, 'Creating Incus snapshot');
+    setJobProgress(job, { percent: 2 });
+    await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
+
+    setJobStep(job, 'Making ZFS snapshot device visible');
+    setJobProgress(job, { percent: 5 });
+    await spawnCommand('zfs', ['set', 'snapdev=visible', zvol], { log: (line) => appendJobLog(job, line) });
+
+    setJobStep(job, 'Waiting for snapshot device');
+    setJobProgress(job, { percent: 8 });
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+
+    setJobStep(job, 'Streaming block device to Restic');
+    const snapshotStream = await deviceReadStream(snapshotDevice);
+    let streamedBytes = 0;
+    const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
+      streamedBytes = currentBytes;
+      setJobProgress(job, {
+        currentBytes,
+        totalBytes: bytesTotal,
+        percent: 10 + percent * 0.78,
+        detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
+      });
+    });
+    try {
+      const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine'], {
+        env: { ...process.env, ...config.resticEnv },
+        input: snapshotStream.pipe(progressStream),
+        log: (line) => appendJobLog(job, line),
+      });
+      const snapshotId = parseResticSnapshotId(result);
+      await verifyResticFileSize(job, snapshotId, `${job.vmName}.raw`, streamedBytes || totalBytes);
+    } finally {
+      if (!snapshotStream.destroyed) {
+        snapshotStream.destroy();
+      }
+      await finished(snapshotStream, { cleanup: true }).catch(() => {});
+    }
+
+    setJobStep(job, 'Hiding ZFS snapshot device');
+    setJobProgress(job, { percent: 90 });
+    await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { log: (line) => appendJobLog(job, line) });
+    await settleUdev(job);
+
+    setJobStep(job, 'Deleting temporary Incus snapshot');
+    setJobProgress(job, { percent: 93 });
+    await deleteIncusSnapshotWithRetry(job, snapshotName);
+
+    setJobStep(job, 'Backing up Incus metadata');
+    setJobProgress(job, { percent: 95 });
+    await backupInstanceMetadata(job, 'virtual-machine');
+
+    setJobStep(job, 'Applying Restic retention policy');
+    setJobProgress(job, { percent: 97 });
+    await runRestic(retentionArgs(job.vmName), {
+      log: (line) => appendJobLog(job, line),
+    });
+
+    finishJob(job, 'success');
+  } catch (error) {
+    appendJobLog(job, error.message);
+    await cleanupBackup(job, zvol, snapshotName);
+    finishJob(job, 'failed', error);
+  }
+}
+
+async function runContainerBackupJob(job) {
+  const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
+  const snapshotName = `s3-backup-${timestamp}`;
+  const dataset = `${config.zfsPoolName}/containers/${job.vmName}`;
+  const snapshot = `${dataset}@snapshot-${snapshotName}`;
+
+  try {
+    const totalBytes = await zfsDatasetUsed(dataset);
+
+    setJobRunning(job, 'Creating Incus snapshot');
+    setJobProgress(job, { percent: 2 });
+    await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
+
+    setJobStep(job, 'Streaming ZFS snapshot to Restic');
+    const zfs = spawn('zfs', ['send', snapshot], { stdio: ['ignore', 'pipe', 'pipe'] });
+    const zfsClosed = waitForProcess(zfs, 'zfs send');
+    zfs.stderr.on('data', (chunk) => appendJobLog(job, chunk.toString().trimEnd()));
+    let streamedBytes = 0;
+    const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
+      streamedBytes = currentBytes;
+      setJobProgress(job, {
+        currentBytes,
+        totalBytes: bytesTotal,
+        percent: 10 + percent * 0.78,
+        detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
+      });
+    });
+    let resticOk = false;
+    try {
+      const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container'], {
+        env: { ...process.env, ...config.resticEnv },
+        input: zfs.stdout.pipe(progressStream),
+        log: (line) => appendJobLog(job, line),
+      });
+      const snapshotId = parseResticSnapshotId(result);
+      await verifyResticFileSize(job, snapshotId, `${job.vmName}.zfs`, streamedBytes || totalBytes);
+      resticOk = true;
+    } finally {
+      if (!resticOk && !zfs.killed) zfs.kill('SIGTERM');
+    }
+    await zfsClosed;
+
+    setJobStep(job, 'Deleting temporary Incus snapshot');
+    setJobProgress(job, { percent: 93 });
+    await deleteIncusSnapshotWithRetry(job, snapshotName);
+
+    setJobStep(job, 'Backing up Incus metadata');
+    setJobProgress(job, { percent: 95 });
+    await backupInstanceMetadata(job, 'container');
+
+    setJobStep(job, 'Applying Restic retention policy');
+    setJobProgress(job, { percent: 97 });
+    await runRestic(retentionArgs(job.vmName), {
+      log: (line) => appendJobLog(job, line),
+    });
+
+    finishJob(job, 'success');
+  } catch (error) {
+    appendJobLog(job, error.message);
+    await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
+    finishJob(job, 'failed', error);
+  }
+}
+
+async function deviceReadStream(path) {
+  const { createReadStream } = await import('node:fs');
+  return createReadStream(path);
+}
+
+async function zfsVolumeSize(zvol) {
+  const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'volsize', zvol]);
+  const size = Number(result.stdout.trim());
+  return Number.isFinite(size) && size > 0 ? size : null;
+}
+
+async function zfsDatasetUsed(dataset) {
+  const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'used', dataset]);
+  const size = Number(result.stdout.trim());
+  return Number.isFinite(size) && size > 0 ? size : null;
+}
+
+async function backupInstanceMetadata(job, instanceType) {
+  const tempDir = await mkdtemp(path.join(tmpdir(), `incus-backup-${job.vmName}-`));
+  try {
+    const metadataDir = path.join(tempDir, job.vmName);
+    await writeMetadataFile(metadataDir, 'README.txt', metadataReadme(job.vmName, instanceType));
+    await writeCommandOutput(metadataDir, 'config.yaml', ['config', 'show', job.vmName]);
+    await writeCommandOutput(metadataDir, 'config-expanded.yaml', ['config', 'show', job.vmName, '--expanded']);
+    await writeCommandOutput(metadataDir, 'info.txt', ['info', job.vmName]);
+    await writeCommandOutput(metadataDir, 'snapshots.json', ['snapshot', 'list', job.vmName, '--format', 'json']);
+    await runRestic(['backup', metadataDir, '--tag', job.vmName, '--tag', 'metadata', '--tag', instanceType], {
+      log: (line) => appendJobLog(job, line),
+    });
+  } finally {
+    await rm(tempDir, { recursive: true, force: true });
+  }
+}
+
+async function writeCommandOutput(directory, filename, args) {
+  const result = await spawnCommand('incus', args);
+  await writeMetadataFile(directory, filename, result.stdout);
+}
+
+async function writeMetadataFile(directory, filename, content) {
+  const { mkdir } = await import('node:fs/promises');
+  await mkdir(directory, { recursive: true });
+  await writeFile(path.join(directory, filename), content || '', { mode: 0o600 });
+}
+
+function metadataReadme(instanceName, instanceType) {
+  return [
+    `Instance: ${instanceName}`,
+    `Type: ${instanceType}`,
+    `Created: ${new Date().toISOString()}`,
+    '',
+    'This directory contains Incus metadata captured alongside the disk/dataset backup.',
+    'It is intended for disaster-recovery reconstruction and restore preflight workflows.',
+    '',
+  ].join('\n');
+}
+
+function waitForProcess(child, label) {
+  return new Promise((resolve, reject) => {
+    child.on('error', reject);
+    child.on('close', (exitCode) => {
+      if (exitCode !== 0 && exitCode !== null) {
+        reject(new Error(`${label} exited with ${exitCode}`));
+        return;
+      }
+      resolve();
+    });
+  });
+}
+
+function formatBytes(value) {
+  if (!value) return '-';
+  const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
+  let size = Number(value);
+  let unitIndex = 0;
+  while (size >= 1024 && unitIndex < units.length - 1) {
+    size /= 1024;
+    unitIndex += 1;
+  }
+  return `${size.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
+}
+
+function retentionArgs(vmName) {
+  const args = ['forget', '--tag', vmName, '--prune'];
+  const keepHourly = positiveInteger(config.retention.keepHourly);
+  const keepDaily = positiveInteger(config.retention.keepDaily);
+  const keepWeekly = positiveInteger(config.retention.keepWeekly);
+  const keepMonthly = positiveInteger(config.retention.keepMonthly);
+
+  if (keepHourly) args.push('--keep-hourly', String(keepHourly));
+  if (keepDaily) args.push('--keep-daily', String(keepDaily));
+  if (keepWeekly) args.push('--keep-weekly', String(keepWeekly));
+  if (keepMonthly) args.push('--keep-monthly', String(keepMonthly));
+  if (!keepHourly && !keepDaily && !keepWeekly && !keepMonthly) args.push('--keep-daily', '7');
+
+  return args;
+}
+
+function positiveInteger(value) {
+  const number = Number(value);
+  return Number.isFinite(number) && number > 0 ? Math.floor(number) : 0;
+}
+
+function parseResticSnapshotId(result) {
+  const output = `${result.stdout || ''}\n${result.stderr || ''}`;
+  const match = output.match(/snapshot\s+([0-9a-f]{8,64})\s+saved/i);
+  if (!match) {
+    throw new Error('Restic backup completed but no snapshot ID could be parsed.');
+  }
+  return match[1];
+}
+
+async function verifyResticFileSize(job, snapshotId, filename, expectedBytes) {
+  if (!expectedBytes) {
+    appendJobLog(job, `Skipping size verification for ${snapshotId}: expected size is unknown.`);
+    return;
+  }
+  try {
+    const storedBytes = await resticSnapshotFileSize(snapshotId, filename);
+    if (storedBytes !== expectedBytes) {
+      throw new Error(`Backup verification failed for ${snapshotId}: stored ${formatBytes(storedBytes)}, expected ${formatBytes(expectedBytes)}.`);
+    }
+    appendJobLog(job, `Verified Restic snapshot ${snapshotId}: ${formatBytes(storedBytes)}.`);
+  } catch (error) {
+    appendJobLog(job, error.message);
+    await forgetFailedSnapshot(job, snapshotId);
+    throw error;
+  }
+}
+
+async function forgetFailedSnapshot(job, snapshotId) {
+  appendJobLog(job, `Removing failed Restic snapshot ${snapshotId}.`);
+  await runRestic(['forget', snapshotId, '--prune'], {
+    log: (line) => appendJobLog(job, line),
+  }).catch((error) => appendJobLog(job, `Failed to remove Restic snapshot ${snapshotId}: ${error.message}`));
+}
+
+async function cleanupBackup(job, zvol, snapshotName) {
+  setJobStep(job, 'Running cleanup');
+  await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], {
+    ignoreExitCode: true,
+    log: (line) => appendJobLog(job, line),
+  }).catch((error) => appendJobLog(job, error.message));
+  await settleUdev(job);
+  await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
+}
+
+async function settleUdev(job) {
+  await spawnCommand('udevadm', ['trigger'], {
+    ignoreExitCode: true,
+    log: (line) => appendJobLog(job, line),
+  }).catch((error) => appendJobLog(job, error.message));
+  await spawnCommand('udevadm', ['settle'], {
+    ignoreExitCode: true,
+    log: (line) => appendJobLog(job, line),
+  }).catch((error) => appendJobLog(job, error.message));
+}
+
+async function deleteIncusSnapshotWithRetry(job, snapshotName, options = {}) {
+  const maxAttempts = 5;
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      return await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], {
+        ignoreExitCode: options.ignoreExitCode,
+        log: (line) => appendJobLog(job, line),
+      });
+    } catch (error) {
+      const retryable = error.message.includes('dataset is busy');
+      if (!retryable || attempt === maxAttempts) {
+        if (options.ignoreExitCode) {
+          appendJobLog(job, error.message);
+          return null;
+        }
+        throw error;
+      }
+      appendJobLog(job, `Snapshot device still busy; retrying delete (${attempt}/${maxAttempts}).`);
+      await delay(2000);
+    }
+  }
+  return null;
+}

agent/src/routes/health.js

@@ -0,0 +1,88 @@
+import { Router } from 'express';
+import { access } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { config, missingEnvVars } from '../config.js';
+import { spawnCommand, runRestic } from '../executor.js';
+
+export const healthRouter = Router();
+
+healthRouter.get('/', async (_req, res) => {
+  const missing = missingEnvVars();
+  const checks = {
+    config: checkValue(!missing.length, missing.length ? `missing: ${missing.join(', ')}` : 'ok'),
+    commands: await checkCommands({
+      incus: ['version'],
+      zfs: ['version'],
+      zpool: ['version'],
+      restic: ['version'],
+      udevadm: ['--version'],
+      dd: ['--version'],
+    }),
+    zfsPool: await checkZfsPool(),
+    zvol: await checkZvolAccess(),
+    resticRepository: await checkResticRepository(),
+  };
+
+  const ok = Object.values(checks).every((value) => value.ok);
+  res.status(ok ? 200 : 503).json({ ok, checks });
+});
+
+async function checkCommands(commands) {
+  const results = {};
+  await Promise.all(Object.entries(commands).map(async ([command, args]) => {
+    results[command] = await checkCommand(command, args);
+  }));
+  const failed = Object.entries(results).filter(([, result]) => !result.ok);
+  return {
+    ok: failed.length === 0,
+    message: failed.length ? `failed: ${failed.map(([command]) => command).join(', ')}` : 'ok',
+    details: results,
+  };
+}
+
+async function checkCommand(command, args) {
+  try {
+    const result = await spawnCommand(command, args, { ignoreExitCode: true });
+    return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || result.stdout.trim() || 'failed');
+  } catch (error) {
+    return checkValue(false, error.message);
+  }
+}
+
+async function checkZfsPool() {
+  if (!config.zfsPoolName) return checkValue(false, 'ZFS_POOL_NAME is not configured');
+  try {
+    const result = await spawnCommand('zpool', ['list', '-Hp', '-o', 'name,size,alloc,free,cap', config.zfsPoolName], { ignoreExitCode: true });
+    if (result.exitCode !== 0) return checkValue(false, result.stderr.trim() || 'zpool list failed');
+    const [name, size, allocated, free, capacity] = result.stdout.trim().split('\t');
+    return {
+      ok: true,
+      message: 'ok',
+      details: { name, size: Number(size), allocated: Number(allocated), free: Number(free), capacity },
+    };
+  } catch (error) {
+    return checkValue(false, error.message);
+  }
+}
+
+async function checkZvolAccess() {
+  try {
+    await access('/dev/zvol', constants.R_OK);
+    return checkValue(true, 'ok');
+  } catch (error) {
+    return checkValue(false, error.message);
+  }
+}
+
+async function checkResticRepository() {
+  try {
+    const result = await runRestic(['--no-lock', 'snapshots', '--json'], { ignoreExitCode: true });
+    return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'restic repository check failed');
+  } catch (error) {
+    return checkValue(false, error.message);
+  }
+}
+
+function checkValue(ok, message, details = null) {
+  return { ok, message, ...(details ? { details } : {}) };
+}

agent/src/routes/jobs.js

@@ -0,0 +1,17 @@
+import { Router } from 'express';
+import { getJob, listJobs } from '../jobs.js';
+
+export const jobsRouter = Router();
+
+jobsRouter.get('/', (_req, res) => {
+  res.json(listJobs());
+});
+
+jobsRouter.get('/:jobId', (req, res) => {
+  const job = getJob(req.params.jobId);
+  if (!job) {
+    res.status(404).json({ error: 'Job not found.' });
+    return;
+  }
+  res.json(job);
+});

agent/src/routes/restore.js

@@ -0,0 +1,201 @@
+import { Router } from 'express';
+import { constants } from 'node:fs';
+import { access } from 'node:fs/promises';
+import { setTimeout as delay } from 'node:timers/promises';
+import { config } from '../config.js';
+import { resticSnapshotFileSize, spawnCommand, streamResticDumpToDd } from '../executor.js';
+import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js';
+import { validateSnapshotForVm, validateVmExists } from '../validators.js';
+
+export const restoreRouter = Router();
+
+restoreRouter.post('/:vmName', async (req, res, next) => {
+  try {
+    const vmName = req.params.vmName;
+    const { snapshotId, confirmVmName } = req.body || {};
+    if (confirmVmName !== vmName) {
+      const error = new Error('Restore confirmation does not match VM name.');
+      error.status = 400;
+      throw error;
+    }
+
+    const instance = await validateVmExists(vmName);
+    if (instance.type === 'container') {
+      const error = new Error('Container restore is not implemented yet. Container backups can be created, but restore needs a safe zfs receive workflow.');
+      error.status = 501;
+      throw error;
+    }
+    const snapshot = await validateSnapshotForVm(vmName, snapshotId);
+    const job = createJob('restore', vmName);
+    runRestoreJob(job, snapshot.id).catch(() => {});
+    res.status(202).json({ jobId: job.id, message: 'Restore job started.' });
+  } catch (error) {
+    next(error);
+  }
+});
+
+async function runRestoreJob(job, snapshotId) {
+  const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
+  const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
+  const tempZvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.restore-${timestamp}.block`;
+  const backupZvol = `${zvol}.pre-restore-${timestamp}`;
+  const failedZvol = `${zvol}.failed-restore-${timestamp}`;
+  const tempDevice = `/dev/zvol/${tempZvol}`;
+  let tempCreated = false;
+  let oldVolumeRenamed = false;
+  let swapped = false;
+
+  try {
+    setJobRunning(job, 'Checking restore size');
+    setJobProgress(job, { percent: 3 });
+    const totalBytes = await zfsVolumeSize(zvol);
+    const resticBytes = await resticSnapshotFileSize(snapshotId, `${job.vmName}.raw`);
+    if (totalBytes && resticBytes && totalBytes !== resticBytes) {
+      throw new Error(`Restore size mismatch: Restic file is ${formatBytes(resticBytes)}, target volume is ${formatBytes(totalBytes)}.`);
+    }
+
+    setJobStep(job, 'Creating staged restore volume');
+    setJobProgress(job, { percent: 6 });
+    await createRestoreVolume(job, zvol, tempZvol, totalBytes || resticBytes);
+    tempCreated = true;
+
+    setJobStep(job, 'Waiting for staged restore device');
+    setJobProgress(job, { percent: 10 });
+    await settleUdev(job);
+    await waitForDevice(tempDevice);
+
+    setJobStep(job, 'Writing Restic snapshot to staged volume');
+    await streamResticDumpToDd(snapshotId, `${job.vmName}.raw`, tempDevice, (line) => appendJobLog(job, line), {
+      totalBytes,
+      onProgress: ({ currentBytes, totalBytes: bytesTotal, percent }) => {
+        setJobProgress(job, {
+          currentBytes,
+          totalBytes: bytesTotal,
+          percent: 12 + percent * 0.72,
+          detail: `Writing ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
+        });
+      },
+    });
+
+    setJobStep(job, 'Preparing staged volume for swap');
+    setJobProgress(job, { percent: 86 });
+    await spawnCommand('zfs', ['set', 'volmode=none', tempZvol], { log: (line) => appendJobLog(job, line) });
+    await settleUdev(job);
+
+    setJobStep(job, 'Stopping VM');
+    setJobProgress(job, { percent: 89 });
+    await spawnCommand('incus', ['stop', job.vmName, '--force'], {
+      ignoreExitCode: true,
+      log: (line) => appendJobLog(job, line),
+    });
+
+    setJobStep(job, 'Swapping restored volume into place');
+    setJobProgress(job, { percent: 92 });
+    await spawnCommand('zfs', ['rename', zvol, backupZvol], { log: (line) => appendJobLog(job, line) });
+    oldVolumeRenamed = true;
+    await spawnCommand('zfs', ['rename', tempZvol, zvol], { log: (line) => appendJobLog(job, line) });
+    tempCreated = false;
+    swapped = true;
+
+    setJobStep(job, 'Starting VM');
+    setJobProgress(job, { percent: 97 });
+    await spawnCommand('incus', ['start', job.vmName], { log: (line) => appendJobLog(job, line) });
+
+    appendJobLog(job, `Pre-restore ZFS volume kept for rollback: ${backupZvol}`);
+    finishJob(job, 'success');
+  } catch (error) {
+    appendJobLog(job, error.message);
+    await cleanupRestoreFailure(job, { zvol, tempZvol, backupZvol, failedZvol, tempCreated, oldVolumeRenamed, swapped });
+    appendJobLog(job, 'VM was not restarted because restore did not complete successfully.');
+    finishJob(job, 'failed', error);
+  }
+}
+
+async function zfsVolumeSize(zvol) {
+  const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'volsize', zvol]);
+  const size = Number(result.stdout.trim());
+  return Number.isFinite(size) && size > 0 ? size : null;
+}
+
+async function zfsProperty(zvol, property) {
+  const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', property, zvol]);
+  return result.stdout.trim();
+}
+
+async function createRestoreVolume(job, sourceZvol, targetZvol, bytes) {
+  if (!bytes) throw new Error('Could not determine restore volume size.');
+  const volblocksize = await zfsProperty(sourceZvol, 'volblocksize');
+  const args = ['create', '-V', `${bytes}B`, '-o', 'volmode=dev'];
+  if (volblocksize) args.push('-o', `volblocksize=${volblocksize}`);
+  args.push(targetZvol);
+  await spawnCommand('zfs', args, { log: (line) => appendJobLog(job, line) });
+}
+
+async function settleUdev(job) {
+  await spawnCommand('udevadm', ['trigger'], {
+    ignoreExitCode: true,
+    log: (line) => appendJobLog(job, line),
+  }).catch((error) => appendJobLog(job, error.message));
+  await spawnCommand('udevadm', ['settle'], {
+    ignoreExitCode: true,
+    log: (line) => appendJobLog(job, line),
+  }).catch((error) => appendJobLog(job, error.message));
+}
+
+async function waitForDevice(devicePath, timeoutMs = 30000) {
+  const started = Date.now();
+  while (Date.now() - started < timeoutMs) {
+    try {
+      await access(devicePath, constants.R_OK | constants.W_OK);
+      return;
+    } catch {
+      await delay(500);
+    }
+  }
+  throw new Error(`Timed out waiting for restore device ${devicePath}.`);
+}
+
+async function cleanupRestoreFailure(job, state) {
+  const { zvol, tempZvol, backupZvol, failedZvol, tempCreated, oldVolumeRenamed, swapped } = state;
+  if (swapped) {
+    setJobStep(job, 'Rolling back swapped restore volume');
+    await spawnCommand('zfs', ['rename', zvol, failedZvol], {
+      ignoreExitCode: true,
+      log: (line) => appendJobLog(job, line),
+    }).catch((cleanupError) => appendJobLog(job, cleanupError.message));
+    await spawnCommand('zfs', ['rename', backupZvol, zvol], {
+      ignoreExitCode: true,
+      log: (line) => appendJobLog(job, line),
+    }).catch((cleanupError) => appendJobLog(job, cleanupError.message));
+    appendJobLog(job, `Rolled back to pre-restore volume ${backupZvol}. Failed restored volume, if present, is ${failedZvol}.`);
+    return;
+  }
+
+  if (oldVolumeRenamed) {
+    setJobStep(job, 'Restoring original volume name after failed swap');
+    await spawnCommand('zfs', ['rename', backupZvol, zvol], {
+      ignoreExitCode: true,
+      log: (line) => appendJobLog(job, line),
+    }).catch((cleanupError) => appendJobLog(job, cleanupError.message));
+  }
+
+  if (tempCreated) {
+    setJobStep(job, 'Removing staged restore volume');
+    await spawnCommand('zfs', ['destroy', '-r', tempZvol], {
+      ignoreExitCode: true,
+      log: (line) => appendJobLog(job, line),
+    }).catch((cleanupError) => appendJobLog(job, cleanupError.message));
+  }
+}
+
+function formatBytes(value) {
+  if (!value) return '-';
+  const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
+  let size = Number(value);
+  let unitIndex = 0;
+  while (size >= 1024 && unitIndex < units.length - 1) {
+    size /= 1024;
+    unitIndex += 1;
+  }
+  return `${size.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
+}

agent/src/routes/schedules.js

@@ -0,0 +1,17 @@
+import { Router } from 'express';
+import { listSchedules, replaceSchedules } from '../scheduler.js';
+
+export const schedulesRouter = Router();
+
+schedulesRouter.get('/', (_req, res) => {
+  res.json(listSchedules());
+});
+
+schedulesRouter.put('/', async (req, res, next) => {
+  try {
+    const schedules = await replaceSchedules(req.body?.schedules || []);
+    res.json(schedules);
+  } catch (error) {
+    next(error);
+  }
+});

agent/src/routes/settings.js

@@ -0,0 +1,23 @@
+import { Router } from 'express';
+import { readEnvSettings, writeEnvSettings } from '../config.js';
+
+export const settingsRouter = Router();
+
+settingsRouter.get('/', async (_req, res, next) => {
+  try {
+    res.json({ fields: await readEnvSettings() });
+  } catch (error) {
+    next(error);
+  }
+});
+
+settingsRouter.put('/', async (req, res, next) => {
+  try {
+    res.json({
+      fields: await writeEnvSettings(req.body?.values || {}),
+      message: 'Settings saved.',
+    });
+  } catch (error) {
+    next(error);
+  }
+});

agent/src/routes/snapshots.js

@@ -0,0 +1,37 @@
+import { Router } from 'express';
+import { runRestic } from '../executor.js';
+import { assertSnapshotIdShape, listSnapshotsForVm, validateSnapshotForVm } from '../validators.js';
+
+export const snapshotsRouter = Router();
+
+snapshotsRouter.get('/:vmName', async (req, res, next) => {
+  try {
+    res.json(await listSnapshotsForVm(req.params.vmName));
+  } catch (error) {
+    next(error);
+  }
+});
+
+snapshotsRouter.get('/:vmName/:snapshotId/files', async (req, res, next) => {
+  try {
+    const snapshot = await validateSnapshotForVm(req.params.vmName, req.params.snapshotId);
+    assertSnapshotIdShape(snapshot.id);
+    const result = await runRestic(['--no-lock', 'ls', '--json', snapshot.id]);
+    const entries = result.stdout
+      .split('\n')
+      .filter(Boolean)
+      .map((line) => JSON.parse(line))
+      .filter((entry) => entry.struct_type === 'node')
+      .map((entry) => ({
+        name: entry.name,
+        path: entry.path,
+        type: entry.type,
+        size: entry.size || 0,
+        mode: entry.mode || '',
+        mtime: entry.mtime || null,
+      }));
+    res.json(entries);
+  } catch (error) {
+    next(error);
+  }
+});

agent/src/routes/vms.js

@@ -0,0 +1,34 @@
+import { Router } from 'express';
+import { activeJobForVm, latestJobForVm } from '../jobs.js';
+import { listIncusVms } from '../validators.js';
+
+export const vmsRouter = Router();
+
+vmsRouter.get('/', async (_req, res, next) => {
+  try {
+    const vms = await listIncusVms();
+    res.json(vms.map((vm) => {
+      const activeJob = activeJobForVm(vm.name);
+      const latestJob = latestJobForVm(vm.name);
+      return {
+        name: vm.name,
+        type: vm.type,
+        status: vm.status,
+        activeJob: activeJob ? summarizeJob(activeJob) : null,
+        lastJobStatus: latestJob?.status || null,
+      };
+    }));
+  } catch (error) {
+    next(error);
+  }
+});
+
+function summarizeJob(job) {
+  return {
+    id: job.id,
+    type: job.type,
+    status: job.status,
+    currentStep: job.currentStep,
+    progress: job.progress,
+  };
+}

agent/src/scheduler.js

@@ -0,0 +1,129 @@
+import crypto from 'node:crypto';
+import { readFile, rename, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { startBackupForVm } from './routes/backup.js';
+
+const schedulesPath = path.resolve(process.cwd(), 'schedules.json');
+const checkIntervalMs = 30 * 1000;
+let schedules = [];
+let timer = null;
+
+export async function startScheduler() {
+  await loadSchedules();
+  if (timer) return;
+  timer = setInterval(runDueSchedules, checkIntervalMs);
+  timer.unref?.();
+  runDueSchedules();
+}
+
+export function listSchedules() {
+  return schedules
+    .map((schedule) => ({ ...schedule }))
+    .sort((a, b) => a.vmName.localeCompare(b.vmName));
+}
+
+export async function replaceSchedules(nextSchedules) {
+  schedules = normalizeSchedules(nextSchedules);
+  await saveSchedules();
+  return listSchedules();
+}
+
+async function runDueSchedules() {
+  const now = new Date();
+  let changed = false;
+
+  for (const schedule of schedules) {
+    if (!schedule.enabled) continue;
+    if (!schedule.nextRunAt) {
+      schedule.nextRunAt = nextRunFrom(now, schedule.intervalHours, schedule.timeOfDay);
+      changed = true;
+      continue;
+    }
+    if (new Date(schedule.nextRunAt) > now) continue;
+
+    try {
+      await startBackupForVm(schedule.vmName);
+      schedule.lastRunAt = now.toISOString();
+      schedule.lastError = '';
+      schedule.nextRunAt = nextRunFrom(now, schedule.intervalHours, schedule.timeOfDay);
+    } catch (error) {
+      schedule.lastError = String(error.message || error);
+      schedule.nextRunAt = new Date(now.getTime() + 5 * 60 * 1000).toISOString();
+    }
+    changed = true;
+  }
+
+  if (changed) {
+    await saveSchedules();
+  }
+}
+
+async function loadSchedules() {
+  try {
+    const content = await readFile(schedulesPath, 'utf8');
+    schedules = normalizeSchedules(JSON.parse(content));
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      schedules = [];
+      return;
+    }
+    throw error;
+  }
+}
+
+async function saveSchedules() {
+  const tempPath = `${schedulesPath}.tmp`;
+  await writeFile(tempPath, `${JSON.stringify(schedules, null, 2)}\n`, { mode: 0o600 });
+  await rename(tempPath, schedulesPath);
+}
+
+function normalizeSchedules(values) {
+  const seen = new Set();
+  return (Array.isArray(values) ? values : [])
+    .map((value) => {
+      const vmName = String(value.vmName || '').trim();
+      const intervalHours = Math.max(1, Number(value.intervalHours) || 24);
+      if (!vmName || seen.has(vmName)) return null;
+      seen.add(vmName);
+      return {
+        id: value.id || `schedule_${crypto.randomBytes(8).toString('hex')}`,
+        vmName,
+        enabled: Boolean(value.enabled),
+        intervalHours,
+        timeOfDay: normalizeTimeOfDay(value.timeOfDay),
+        nextRunAt: value.nextRunAt || nextRunFrom(new Date(), intervalHours, normalizeTimeOfDay(value.timeOfDay)),
+        lastRunAt: value.lastRunAt || null,
+        lastError: value.lastError || '',
+      };
+    })
+    .filter(Boolean);
+}
+
+function nextRunFrom(date, intervalHours, timeOfDay) {
+  const intervalMs = intervalHours * 60 * 60 * 1000;
+  const candidate = withTimeOfDay(date, timeOfDay);
+
+  while (candidate <= date) {
+    candidate.setTime(candidate.getTime() + intervalMs);
+  }
+
+  return candidate.toISOString();
+}
+
+function withTimeOfDay(date, timeOfDay) {
+  const [hours, minutes] = normalizeTimeOfDay(timeOfDay).split(':').map(Number);
+  const next = new Date(date);
+  next.setHours(hours, minutes, 0, 0);
+  return next;
+}
+
+function normalizeTimeOfDay(value) {
+  const text = String(value || '02:00').trim();
+  if (/^\d{2}:\d{2}$/.test(text)) {
+    const [hours, minutes] = text.split(':').map(Number);
+    if (hours >= 0 && hours <= 23 && minutes >= 0 && minutes <= 59) {
+      return text;
+    }
+  }
+  return '02:00';
+}

agent/src/validators.js

@@ -0,0 +1,73 @@
+import { spawnCommand, runRestic } from './executor.js';
+
+const vmNamePattern = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/;
+const snapshotIdPattern = /^[A-Fa-f0-9]{8,64}$/;
+
+export function assertVmNameShape(vmName) {
+  if (!vmNamePattern.test(vmName || '')) {
+    const error = new Error('Invalid VM name.');
+    error.status = 400;
+    throw error;
+  }
+}
+
+export function assertSnapshotIdShape(snapshotId) {
+  if (snapshotId === 'latest') return;
+  if (!snapshotIdPattern.test(snapshotId || '')) {
+    const error = new Error('Invalid snapshot id.');
+    error.status = 400;
+    throw error;
+  }
+}
+
+export async function listIncusInstances() {
+  const result = await spawnCommand('incus', ['list', '--format', 'json']);
+  const entries = JSON.parse(result.stdout || '[]');
+  return entries.filter((entry) => ['virtual-machine', 'container'].includes(entry.type));
+}
+
+export async function listIncusVms() {
+  return listIncusInstances();
+}
+
+export async function validateVmExists(instanceName) {
+  assertVmNameShape(instanceName);
+  const instances = await listIncusInstances();
+  const instance = instances.find((entry) => entry.name === instanceName);
+  if (!instance) {
+    const error = new Error(`Instance "${instanceName}" was not found.`);
+    error.status = 404;
+    throw error;
+  }
+  return instance;
+}
+
+export async function listSnapshotsForVm(vmName) {
+  await validateVmExists(vmName);
+  const result = await runRestic(['--no-lock', 'snapshots', '--json', '--tag', vmName]);
+  const snapshots = JSON.parse(result.stdout || '[]');
+  return snapshots
+    .filter((snapshot) => !(snapshot.tags || []).includes('metadata'))
+    .sort((a, b) => String(b.time || '').localeCompare(String(a.time || '')));
+}
+
+export async function validateSnapshotForVm(vmName, snapshotId) {
+  assertSnapshotIdShape(snapshotId);
+  const snapshots = await listSnapshotsForVm(vmName);
+  if (snapshotId === 'latest') {
+    if (!snapshots.length) {
+      const error = new Error(`No snapshots found for ${vmName}.`);
+      error.status = 404;
+      throw error;
+    }
+    return snapshots[0];
+  }
+
+  const snapshot = snapshots.find((entry) => entry.id?.startsWith(snapshotId));
+  if (!snapshot) {
+    const error = new Error(`Snapshot "${snapshotId}" was not found for ${vmName}.`);
+    error.status = 404;
+    throw error;
+  }
+  return snapshot;
+}