added install script for agent
changed backend to agent
This commit is contained in:
Philipp
@@ -0,0 +1,358 @@
|
||||
import { Router } from 'express';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { finished } from 'node:stream/promises';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { config } from '../config.js';
|
||||
import { createProgressStream, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js';
|
||||
import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js';
|
||||
import { validateVmExists } from '../validators.js';
|
||||
|
||||
export const backupRouter = Router();
|
||||
|
||||
backupRouter.post('/:vmName', async (req, res, next) => {
|
||||
try {
|
||||
const vmName = req.params.vmName;
|
||||
const job = await startBackupForVm(vmName);
|
||||
res.status(202).json({ jobId: job.id, message: 'Backup job started.' });
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
export async function startBackupForVm(vmName) {
|
||||
const instance = await validateVmExists(vmName);
|
||||
const job = createJob('backup', vmName);
|
||||
job.instanceType = instance.type;
|
||||
runBackupJob(job, instance).catch(() => {});
|
||||
return job;
|
||||
}
|
||||
|
||||
export async function runBackupJob(job, instance = null) {
|
||||
const currentInstance = instance || await validateVmExists(job.vmName);
|
||||
if (currentInstance.type === 'container') {
|
||||
await runContainerBackupJob(job);
|
||||
return;
|
||||
}
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
||||
const snapshotName = `s3-backup-${timestamp}`;
|
||||
const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
|
||||
const snapshotDevice = `/dev/zvol/${zvol}@snapshot-${snapshotName}`;
|
||||
|
||||
try {
|
||||
const totalBytes = await zfsVolumeSize(zvol);
|
||||
|
||||
setJobRunning(job, 'Creating Incus snapshot');
|
||||
setJobProgress(job, { percent: 2 });
|
||||
await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
|
||||
|
||||
setJobStep(job, 'Making ZFS snapshot device visible');
|
||||
setJobProgress(job, { percent: 5 });
|
||||
await spawnCommand('zfs', ['set', 'snapdev=visible', zvol], { log: (line) => appendJobLog(job, line) });
|
||||
|
||||
setJobStep(job, 'Waiting for snapshot device');
|
||||
setJobProgress(job, { percent: 8 });
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
|
||||
setJobStep(job, 'Streaming block device to Restic');
|
||||
const snapshotStream = await deviceReadStream(snapshotDevice);
|
||||
let streamedBytes = 0;
|
||||
const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
|
||||
streamedBytes = currentBytes;
|
||||
setJobProgress(job, {
|
||||
currentBytes,
|
||||
totalBytes: bytesTotal,
|
||||
percent: 10 + percent * 0.78,
|
||||
detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
|
||||
});
|
||||
});
|
||||
try {
|
||||
const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine'], {
|
||||
env: { ...process.env, ...config.resticEnv },
|
||||
input: snapshotStream.pipe(progressStream),
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
const snapshotId = parseResticSnapshotId(result);
|
||||
await verifyResticFileSize(job, snapshotId, `${job.vmName}.raw`, streamedBytes || totalBytes);
|
||||
} finally {
|
||||
if (!snapshotStream.destroyed) {
|
||||
snapshotStream.destroy();
|
||||
}
|
||||
await finished(snapshotStream, { cleanup: true }).catch(() => {});
|
||||
}
|
||||
|
||||
setJobStep(job, 'Hiding ZFS snapshot device');
|
||||
setJobProgress(job, { percent: 90 });
|
||||
await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { log: (line) => appendJobLog(job, line) });
|
||||
await settleUdev(job);
|
||||
|
||||
setJobStep(job, 'Deleting temporary Incus snapshot');
|
||||
setJobProgress(job, { percent: 93 });
|
||||
await deleteIncusSnapshotWithRetry(job, snapshotName);
|
||||
|
||||
setJobStep(job, 'Backing up Incus metadata');
|
||||
setJobProgress(job, { percent: 95 });
|
||||
await backupInstanceMetadata(job, 'virtual-machine');
|
||||
|
||||
setJobStep(job, 'Applying Restic retention policy');
|
||||
setJobProgress(job, { percent: 97 });
|
||||
await runRestic(retentionArgs(job.vmName), {
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
|
||||
finishJob(job, 'success');
|
||||
} catch (error) {
|
||||
appendJobLog(job, error.message);
|
||||
await cleanupBackup(job, zvol, snapshotName);
|
||||
finishJob(job, 'failed', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function runContainerBackupJob(job) {
|
||||
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
||||
const snapshotName = `s3-backup-${timestamp}`;
|
||||
const dataset = `${config.zfsPoolName}/containers/${job.vmName}`;
|
||||
const snapshot = `${dataset}@snapshot-${snapshotName}`;
|
||||
|
||||
try {
|
||||
const totalBytes = await zfsDatasetUsed(dataset);
|
||||
|
||||
setJobRunning(job, 'Creating Incus snapshot');
|
||||
setJobProgress(job, { percent: 2 });
|
||||
await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
|
||||
|
||||
setJobStep(job, 'Streaming ZFS snapshot to Restic');
|
||||
const zfs = spawn('zfs', ['send', snapshot], { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
const zfsClosed = waitForProcess(zfs, 'zfs send');
|
||||
zfs.stderr.on('data', (chunk) => appendJobLog(job, chunk.toString().trimEnd()));
|
||||
let streamedBytes = 0;
|
||||
const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
|
||||
streamedBytes = currentBytes;
|
||||
setJobProgress(job, {
|
||||
currentBytes,
|
||||
totalBytes: bytesTotal,
|
||||
percent: 10 + percent * 0.78,
|
||||
detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
|
||||
});
|
||||
});
|
||||
let resticOk = false;
|
||||
try {
|
||||
const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container'], {
|
||||
env: { ...process.env, ...config.resticEnv },
|
||||
input: zfs.stdout.pipe(progressStream),
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
const snapshotId = parseResticSnapshotId(result);
|
||||
await verifyResticFileSize(job, snapshotId, `${job.vmName}.zfs`, streamedBytes || totalBytes);
|
||||
resticOk = true;
|
||||
} finally {
|
||||
if (!resticOk && !zfs.killed) zfs.kill('SIGTERM');
|
||||
}
|
||||
await zfsClosed;
|
||||
|
||||
setJobStep(job, 'Deleting temporary Incus snapshot');
|
||||
setJobProgress(job, { percent: 93 });
|
||||
await deleteIncusSnapshotWithRetry(job, snapshotName);
|
||||
|
||||
setJobStep(job, 'Backing up Incus metadata');
|
||||
setJobProgress(job, { percent: 95 });
|
||||
await backupInstanceMetadata(job, 'container');
|
||||
|
||||
setJobStep(job, 'Applying Restic retention policy');
|
||||
setJobProgress(job, { percent: 97 });
|
||||
await runRestic(retentionArgs(job.vmName), {
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
|
||||
finishJob(job, 'success');
|
||||
} catch (error) {
|
||||
appendJobLog(job, error.message);
|
||||
await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
|
||||
finishJob(job, 'failed', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function deviceReadStream(path) {
|
||||
const { createReadStream } = await import('node:fs');
|
||||
return createReadStream(path);
|
||||
}
|
||||
|
||||
async function zfsVolumeSize(zvol) {
|
||||
const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'volsize', zvol]);
|
||||
const size = Number(result.stdout.trim());
|
||||
return Number.isFinite(size) && size > 0 ? size : null;
|
||||
}
|
||||
|
||||
async function zfsDatasetUsed(dataset) {
|
||||
const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'used', dataset]);
|
||||
const size = Number(result.stdout.trim());
|
||||
return Number.isFinite(size) && size > 0 ? size : null;
|
||||
}
|
||||
|
||||
async function backupInstanceMetadata(job, instanceType) {
|
||||
const tempDir = await mkdtemp(path.join(tmpdir(), `incus-backup-${job.vmName}-`));
|
||||
try {
|
||||
const metadataDir = path.join(tempDir, job.vmName);
|
||||
await writeMetadataFile(metadataDir, 'README.txt', metadataReadme(job.vmName, instanceType));
|
||||
await writeCommandOutput(metadataDir, 'config.yaml', ['config', 'show', job.vmName]);
|
||||
await writeCommandOutput(metadataDir, 'config-expanded.yaml', ['config', 'show', job.vmName, '--expanded']);
|
||||
await writeCommandOutput(metadataDir, 'info.txt', ['info', job.vmName]);
|
||||
await writeCommandOutput(metadataDir, 'snapshots.json', ['snapshot', 'list', job.vmName, '--format', 'json']);
|
||||
await runRestic(['backup', metadataDir, '--tag', job.vmName, '--tag', 'metadata', '--tag', instanceType], {
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function writeCommandOutput(directory, filename, args) {
|
||||
const result = await spawnCommand('incus', args);
|
||||
await writeMetadataFile(directory, filename, result.stdout);
|
||||
}
|
||||
|
||||
async function writeMetadataFile(directory, filename, content) {
|
||||
const { mkdir } = await import('node:fs/promises');
|
||||
await mkdir(directory, { recursive: true });
|
||||
await writeFile(path.join(directory, filename), content || '', { mode: 0o600 });
|
||||
}
|
||||
|
||||
function metadataReadme(instanceName, instanceType) {
|
||||
return [
|
||||
`Instance: ${instanceName}`,
|
||||
`Type: ${instanceType}`,
|
||||
`Created: ${new Date().toISOString()}`,
|
||||
'',
|
||||
'This directory contains Incus metadata captured alongside the disk/dataset backup.',
|
||||
'It is intended for disaster-recovery reconstruction and restore preflight workflows.',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function waitForProcess(child, label) {
|
||||
return new Promise((resolve, reject) => {
|
||||
child.on('error', reject);
|
||||
child.on('close', (exitCode) => {
|
||||
if (exitCode !== 0 && exitCode !== null) {
|
||||
reject(new Error(`${label} exited with ${exitCode}`));
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function formatBytes(value) {
|
||||
if (!value) return '-';
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||
let size = Number(value);
|
||||
let unitIndex = 0;
|
||||
while (size >= 1024 && unitIndex < units.length - 1) {
|
||||
size /= 1024;
|
||||
unitIndex += 1;
|
||||
}
|
||||
return `${size.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
|
||||
}
|
||||
|
||||
function retentionArgs(vmName) {
|
||||
const args = ['forget', '--tag', vmName, '--prune'];
|
||||
const keepHourly = positiveInteger(config.retention.keepHourly);
|
||||
const keepDaily = positiveInteger(config.retention.keepDaily);
|
||||
const keepWeekly = positiveInteger(config.retention.keepWeekly);
|
||||
const keepMonthly = positiveInteger(config.retention.keepMonthly);
|
||||
|
||||
if (keepHourly) args.push('--keep-hourly', String(keepHourly));
|
||||
if (keepDaily) args.push('--keep-daily', String(keepDaily));
|
||||
if (keepWeekly) args.push('--keep-weekly', String(keepWeekly));
|
||||
if (keepMonthly) args.push('--keep-monthly', String(keepMonthly));
|
||||
if (!keepHourly && !keepDaily && !keepWeekly && !keepMonthly) args.push('--keep-daily', '7');
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
function positiveInteger(value) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) && number > 0 ? Math.floor(number) : 0;
|
||||
}
|
||||
|
||||
function parseResticSnapshotId(result) {
|
||||
const output = `${result.stdout || ''}\n${result.stderr || ''}`;
|
||||
const match = output.match(/snapshot\s+([0-9a-f]{8,64})\s+saved/i);
|
||||
if (!match) {
|
||||
throw new Error('Restic backup completed but no snapshot ID could be parsed.');
|
||||
}
|
||||
return match[1];
|
||||
}
|
||||
|
||||
async function verifyResticFileSize(job, snapshotId, filename, expectedBytes) {
|
||||
if (!expectedBytes) {
|
||||
appendJobLog(job, `Skipping size verification for ${snapshotId}: expected size is unknown.`);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const storedBytes = await resticSnapshotFileSize(snapshotId, filename);
|
||||
if (storedBytes !== expectedBytes) {
|
||||
throw new Error(`Backup verification failed for ${snapshotId}: stored ${formatBytes(storedBytes)}, expected ${formatBytes(expectedBytes)}.`);
|
||||
}
|
||||
appendJobLog(job, `Verified Restic snapshot ${snapshotId}: ${formatBytes(storedBytes)}.`);
|
||||
} catch (error) {
|
||||
appendJobLog(job, error.message);
|
||||
await forgetFailedSnapshot(job, snapshotId);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function forgetFailedSnapshot(job, snapshotId) {
|
||||
appendJobLog(job, `Removing failed Restic snapshot ${snapshotId}.`);
|
||||
await runRestic(['forget', snapshotId, '--prune'], {
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((error) => appendJobLog(job, `Failed to remove Restic snapshot ${snapshotId}: ${error.message}`));
|
||||
}
|
||||
|
||||
async function cleanupBackup(job, zvol, snapshotName) {
|
||||
setJobStep(job, 'Running cleanup');
|
||||
await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((error) => appendJobLog(job, error.message));
|
||||
await settleUdev(job);
|
||||
await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
|
||||
}
|
||||
|
||||
async function settleUdev(job) {
|
||||
await spawnCommand('udevadm', ['trigger'], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((error) => appendJobLog(job, error.message));
|
||||
await spawnCommand('udevadm', ['settle'], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((error) => appendJobLog(job, error.message));
|
||||
}
|
||||
|
||||
async function deleteIncusSnapshotWithRetry(job, snapshotName, options = {}) {
|
||||
const maxAttempts = 5;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||
try {
|
||||
return await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], {
|
||||
ignoreExitCode: options.ignoreExitCode,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
} catch (error) {
|
||||
const retryable = error.message.includes('dataset is busy');
|
||||
if (!retryable || attempt === maxAttempts) {
|
||||
if (options.ignoreExitCode) {
|
||||
appendJobLog(job, error.message);
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
appendJobLog(job, `Snapshot device still busy; retrying delete (${attempt}/${maxAttempts}).`);
|
||||
await delay(2000);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import { Router } from 'express';
|
||||
import { access } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import { config, missingEnvVars } from '../config.js';
|
||||
import { spawnCommand, runRestic } from '../executor.js';
|
||||
|
||||
export const healthRouter = Router();
|
||||
|
||||
healthRouter.get('/', async (_req, res) => {
|
||||
const missing = missingEnvVars();
|
||||
const checks = {
|
||||
config: checkValue(!missing.length, missing.length ? `missing: ${missing.join(', ')}` : 'ok'),
|
||||
commands: await checkCommands({
|
||||
incus: ['version'],
|
||||
zfs: ['version'],
|
||||
zpool: ['version'],
|
||||
restic: ['version'],
|
||||
udevadm: ['--version'],
|
||||
dd: ['--version'],
|
||||
}),
|
||||
zfsPool: await checkZfsPool(),
|
||||
zvol: await checkZvolAccess(),
|
||||
resticRepository: await checkResticRepository(),
|
||||
};
|
||||
|
||||
const ok = Object.values(checks).every((value) => value.ok);
|
||||
res.status(ok ? 200 : 503).json({ ok, checks });
|
||||
});
|
||||
|
||||
async function checkCommands(commands) {
|
||||
const results = {};
|
||||
await Promise.all(Object.entries(commands).map(async ([command, args]) => {
|
||||
results[command] = await checkCommand(command, args);
|
||||
}));
|
||||
const failed = Object.entries(results).filter(([, result]) => !result.ok);
|
||||
return {
|
||||
ok: failed.length === 0,
|
||||
message: failed.length ? `failed: ${failed.map(([command]) => command).join(', ')}` : 'ok',
|
||||
details: results,
|
||||
};
|
||||
}
|
||||
|
||||
async function checkCommand(command, args) {
|
||||
try {
|
||||
const result = await spawnCommand(command, args, { ignoreExitCode: true });
|
||||
return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || result.stdout.trim() || 'failed');
|
||||
} catch (error) {
|
||||
return checkValue(false, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function checkZfsPool() {
|
||||
if (!config.zfsPoolName) return checkValue(false, 'ZFS_POOL_NAME is not configured');
|
||||
try {
|
||||
const result = await spawnCommand('zpool', ['list', '-Hp', '-o', 'name,size,alloc,free,cap', config.zfsPoolName], { ignoreExitCode: true });
|
||||
if (result.exitCode !== 0) return checkValue(false, result.stderr.trim() || 'zpool list failed');
|
||||
const [name, size, allocated, free, capacity] = result.stdout.trim().split('\t');
|
||||
return {
|
||||
ok: true,
|
||||
message: 'ok',
|
||||
details: { name, size: Number(size), allocated: Number(allocated), free: Number(free), capacity },
|
||||
};
|
||||
} catch (error) {
|
||||
return checkValue(false, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function checkZvolAccess() {
|
||||
try {
|
||||
await access('/dev/zvol', constants.R_OK);
|
||||
return checkValue(true, 'ok');
|
||||
} catch (error) {
|
||||
return checkValue(false, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function checkResticRepository() {
|
||||
try {
|
||||
const result = await runRestic(['--no-lock', 'snapshots', '--json'], { ignoreExitCode: true });
|
||||
return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'restic repository check failed');
|
||||
} catch (error) {
|
||||
return checkValue(false, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
function checkValue(ok, message, details = null) {
|
||||
return { ok, message, ...(details ? { details } : {}) };
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Router } from 'express';
|
||||
import { getJob, listJobs } from '../jobs.js';
|
||||
|
||||
export const jobsRouter = Router();
|
||||
|
||||
jobsRouter.get('/', (_req, res) => {
|
||||
res.json(listJobs());
|
||||
});
|
||||
|
||||
jobsRouter.get('/:jobId', (req, res) => {
|
||||
const job = getJob(req.params.jobId);
|
||||
if (!job) {
|
||||
res.status(404).json({ error: 'Job not found.' });
|
||||
return;
|
||||
}
|
||||
res.json(job);
|
||||
});
|
||||
@@ -0,0 +1,201 @@
|
||||
import { Router } from 'express';
|
||||
import { constants } from 'node:fs';
|
||||
import { access } from 'node:fs/promises';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { config } from '../config.js';
|
||||
import { resticSnapshotFileSize, spawnCommand, streamResticDumpToDd } from '../executor.js';
|
||||
import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js';
|
||||
import { validateSnapshotForVm, validateVmExists } from '../validators.js';
|
||||
|
||||
export const restoreRouter = Router();
|
||||
|
||||
restoreRouter.post('/:vmName', async (req, res, next) => {
|
||||
try {
|
||||
const vmName = req.params.vmName;
|
||||
const { snapshotId, confirmVmName } = req.body || {};
|
||||
if (confirmVmName !== vmName) {
|
||||
const error = new Error('Restore confirmation does not match VM name.');
|
||||
error.status = 400;
|
||||
throw error;
|
||||
}
|
||||
|
||||
const instance = await validateVmExists(vmName);
|
||||
if (instance.type === 'container') {
|
||||
const error = new Error('Container restore is not implemented yet. Container backups can be created, but restore needs a safe zfs receive workflow.');
|
||||
error.status = 501;
|
||||
throw error;
|
||||
}
|
||||
const snapshot = await validateSnapshotForVm(vmName, snapshotId);
|
||||
const job = createJob('restore', vmName);
|
||||
runRestoreJob(job, snapshot.id).catch(() => {});
|
||||
res.status(202).json({ jobId: job.id, message: 'Restore job started.' });
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
async function runRestoreJob(job, snapshotId) {
|
||||
const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
|
||||
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
||||
const tempZvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.restore-${timestamp}.block`;
|
||||
const backupZvol = `${zvol}.pre-restore-${timestamp}`;
|
||||
const failedZvol = `${zvol}.failed-restore-${timestamp}`;
|
||||
const tempDevice = `/dev/zvol/${tempZvol}`;
|
||||
let tempCreated = false;
|
||||
let oldVolumeRenamed = false;
|
||||
let swapped = false;
|
||||
|
||||
try {
|
||||
setJobRunning(job, 'Checking restore size');
|
||||
setJobProgress(job, { percent: 3 });
|
||||
const totalBytes = await zfsVolumeSize(zvol);
|
||||
const resticBytes = await resticSnapshotFileSize(snapshotId, `${job.vmName}.raw`);
|
||||
if (totalBytes && resticBytes && totalBytes !== resticBytes) {
|
||||
throw new Error(`Restore size mismatch: Restic file is ${formatBytes(resticBytes)}, target volume is ${formatBytes(totalBytes)}.`);
|
||||
}
|
||||
|
||||
setJobStep(job, 'Creating staged restore volume');
|
||||
setJobProgress(job, { percent: 6 });
|
||||
await createRestoreVolume(job, zvol, tempZvol, totalBytes || resticBytes);
|
||||
tempCreated = true;
|
||||
|
||||
setJobStep(job, 'Waiting for staged restore device');
|
||||
setJobProgress(job, { percent: 10 });
|
||||
await settleUdev(job);
|
||||
await waitForDevice(tempDevice);
|
||||
|
||||
setJobStep(job, 'Writing Restic snapshot to staged volume');
|
||||
await streamResticDumpToDd(snapshotId, `${job.vmName}.raw`, tempDevice, (line) => appendJobLog(job, line), {
|
||||
totalBytes,
|
||||
onProgress: ({ currentBytes, totalBytes: bytesTotal, percent }) => {
|
||||
setJobProgress(job, {
|
||||
currentBytes,
|
||||
totalBytes: bytesTotal,
|
||||
percent: 12 + percent * 0.72,
|
||||
detail: `Writing ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
setJobStep(job, 'Preparing staged volume for swap');
|
||||
setJobProgress(job, { percent: 86 });
|
||||
await spawnCommand('zfs', ['set', 'volmode=none', tempZvol], { log: (line) => appendJobLog(job, line) });
|
||||
await settleUdev(job);
|
||||
|
||||
setJobStep(job, 'Stopping VM');
|
||||
setJobProgress(job, { percent: 89 });
|
||||
await spawnCommand('incus', ['stop', job.vmName, '--force'], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
|
||||
setJobStep(job, 'Swapping restored volume into place');
|
||||
setJobProgress(job, { percent: 92 });
|
||||
await spawnCommand('zfs', ['rename', zvol, backupZvol], { log: (line) => appendJobLog(job, line) });
|
||||
oldVolumeRenamed = true;
|
||||
await spawnCommand('zfs', ['rename', tempZvol, zvol], { log: (line) => appendJobLog(job, line) });
|
||||
tempCreated = false;
|
||||
swapped = true;
|
||||
|
||||
setJobStep(job, 'Starting VM');
|
||||
setJobProgress(job, { percent: 97 });
|
||||
await spawnCommand('incus', ['start', job.vmName], { log: (line) => appendJobLog(job, line) });
|
||||
|
||||
appendJobLog(job, `Pre-restore ZFS volume kept for rollback: ${backupZvol}`);
|
||||
finishJob(job, 'success');
|
||||
} catch (error) {
|
||||
appendJobLog(job, error.message);
|
||||
await cleanupRestoreFailure(job, { zvol, tempZvol, backupZvol, failedZvol, tempCreated, oldVolumeRenamed, swapped });
|
||||
appendJobLog(job, 'VM was not restarted because restore did not complete successfully.');
|
||||
finishJob(job, 'failed', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function zfsVolumeSize(zvol) {
|
||||
const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'volsize', zvol]);
|
||||
const size = Number(result.stdout.trim());
|
||||
return Number.isFinite(size) && size > 0 ? size : null;
|
||||
}
|
||||
|
||||
async function zfsProperty(zvol, property) {
|
||||
const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', property, zvol]);
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
async function createRestoreVolume(job, sourceZvol, targetZvol, bytes) {
|
||||
if (!bytes) throw new Error('Could not determine restore volume size.');
|
||||
const volblocksize = await zfsProperty(sourceZvol, 'volblocksize');
|
||||
const args = ['create', '-V', `${bytes}B`, '-o', 'volmode=dev'];
|
||||
if (volblocksize) args.push('-o', `volblocksize=${volblocksize}`);
|
||||
args.push(targetZvol);
|
||||
await spawnCommand('zfs', args, { log: (line) => appendJobLog(job, line) });
|
||||
}
|
||||
|
||||
async function settleUdev(job) {
|
||||
await spawnCommand('udevadm', ['trigger'], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((error) => appendJobLog(job, error.message));
|
||||
await spawnCommand('udevadm', ['settle'], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((error) => appendJobLog(job, error.message));
|
||||
}
|
||||
|
||||
async function waitForDevice(devicePath, timeoutMs = 30000) {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
try {
|
||||
await access(devicePath, constants.R_OK | constants.W_OK);
|
||||
return;
|
||||
} catch {
|
||||
await delay(500);
|
||||
}
|
||||
}
|
||||
throw new Error(`Timed out waiting for restore device ${devicePath}.`);
|
||||
}
|
||||
|
||||
async function cleanupRestoreFailure(job, state) {
|
||||
const { zvol, tempZvol, backupZvol, failedZvol, tempCreated, oldVolumeRenamed, swapped } = state;
|
||||
if (swapped) {
|
||||
setJobStep(job, 'Rolling back swapped restore volume');
|
||||
await spawnCommand('zfs', ['rename', zvol, failedZvol], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((cleanupError) => appendJobLog(job, cleanupError.message));
|
||||
await spawnCommand('zfs', ['rename', backupZvol, zvol], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((cleanupError) => appendJobLog(job, cleanupError.message));
|
||||
appendJobLog(job, `Rolled back to pre-restore volume ${backupZvol}. Failed restored volume, if present, is ${failedZvol}.`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (oldVolumeRenamed) {
|
||||
setJobStep(job, 'Restoring original volume name after failed swap');
|
||||
await spawnCommand('zfs', ['rename', backupZvol, zvol], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((cleanupError) => appendJobLog(job, cleanupError.message));
|
||||
}
|
||||
|
||||
if (tempCreated) {
|
||||
setJobStep(job, 'Removing staged restore volume');
|
||||
await spawnCommand('zfs', ['destroy', '-r', tempZvol], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((cleanupError) => appendJobLog(job, cleanupError.message));
|
||||
}
|
||||
}
|
||||
|
||||
function formatBytes(value) {
|
||||
if (!value) return '-';
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||
let size = Number(value);
|
||||
let unitIndex = 0;
|
||||
while (size >= 1024 && unitIndex < units.length - 1) {
|
||||
size /= 1024;
|
||||
unitIndex += 1;
|
||||
}
|
||||
return `${size.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Router } from 'express';
|
||||
import { listSchedules, replaceSchedules } from '../scheduler.js';
|
||||
|
||||
export const schedulesRouter = Router();
|
||||
|
||||
schedulesRouter.get('/', (_req, res) => {
|
||||
res.json(listSchedules());
|
||||
});
|
||||
|
||||
schedulesRouter.put('/', async (req, res, next) => {
|
||||
try {
|
||||
const schedules = await replaceSchedules(req.body?.schedules || []);
|
||||
res.json(schedules);
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,23 @@
|
||||
import { Router } from 'express';
|
||||
import { readEnvSettings, writeEnvSettings } from '../config.js';
|
||||
|
||||
export const settingsRouter = Router();
|
||||
|
||||
settingsRouter.get('/', async (_req, res, next) => {
|
||||
try {
|
||||
res.json({ fields: await readEnvSettings() });
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
settingsRouter.put('/', async (req, res, next) => {
|
||||
try {
|
||||
res.json({
|
||||
fields: await writeEnvSettings(req.body?.values || {}),
|
||||
message: 'Settings saved.',
|
||||
});
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,37 @@
|
||||
import { Router } from 'express';
|
||||
import { runRestic } from '../executor.js';
|
||||
import { assertSnapshotIdShape, listSnapshotsForVm, validateSnapshotForVm } from '../validators.js';
|
||||
|
||||
export const snapshotsRouter = Router();
|
||||
|
||||
snapshotsRouter.get('/:vmName', async (req, res, next) => {
|
||||
try {
|
||||
res.json(await listSnapshotsForVm(req.params.vmName));
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
snapshotsRouter.get('/:vmName/:snapshotId/files', async (req, res, next) => {
|
||||
try {
|
||||
const snapshot = await validateSnapshotForVm(req.params.vmName, req.params.snapshotId);
|
||||
assertSnapshotIdShape(snapshot.id);
|
||||
const result = await runRestic(['--no-lock', 'ls', '--json', snapshot.id]);
|
||||
const entries = result.stdout
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line))
|
||||
.filter((entry) => entry.struct_type === 'node')
|
||||
.map((entry) => ({
|
||||
name: entry.name,
|
||||
path: entry.path,
|
||||
type: entry.type,
|
||||
size: entry.size || 0,
|
||||
mode: entry.mode || '',
|
||||
mtime: entry.mtime || null,
|
||||
}));
|
||||
res.json(entries);
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
import { Router } from 'express';
|
||||
import { activeJobForVm, latestJobForVm } from '../jobs.js';
|
||||
import { listIncusVms } from '../validators.js';
|
||||
|
||||
export const vmsRouter = Router();
|
||||
|
||||
vmsRouter.get('/', async (_req, res, next) => {
|
||||
try {
|
||||
const vms = await listIncusVms();
|
||||
res.json(vms.map((vm) => {
|
||||
const activeJob = activeJobForVm(vm.name);
|
||||
const latestJob = latestJobForVm(vm.name);
|
||||
return {
|
||||
name: vm.name,
|
||||
type: vm.type,
|
||||
status: vm.status,
|
||||
activeJob: activeJob ? summarizeJob(activeJob) : null,
|
||||
lastJobStatus: latestJob?.status || null,
|
||||
};
|
||||
}));
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
function summarizeJob(job) {
|
||||
return {
|
||||
id: job.id,
|
||||
type: job.type,
|
||||
status: job.status,
|
||||
currentStep: job.currentStep,
|
||||
progress: job.progress,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user