added support for container Backup
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import { Router } from 'express';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { finished } from 'node:stream/promises';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { config } from '../config.js';
|
||||
@@ -19,13 +20,20 @@ backupRouter.post('/:vmName', async (req, res, next) => {
|
||||
});
|
||||
|
||||
export async function startBackupForVm(vmName) {
|
||||
await validateVmExists(vmName);
|
||||
const instance = await validateVmExists(vmName);
|
||||
const job = createJob('backup', vmName);
|
||||
runBackupJob(job).catch(() => {});
|
||||
job.instanceType = instance.type;
|
||||
runBackupJob(job, instance).catch(() => {});
|
||||
return job;
|
||||
}
|
||||
|
||||
export async function runBackupJob(job) {
|
||||
export async function runBackupJob(job, instance = null) {
|
||||
const currentInstance = instance || await validateVmExists(job.vmName);
|
||||
if (currentInstance.type === 'container') {
|
||||
await runContainerBackupJob(job);
|
||||
return;
|
||||
}
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
||||
const snapshotName = `s3-backup-${timestamp}`;
|
||||
const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
|
||||
@@ -92,6 +100,62 @@ export async function runBackupJob(job) {
|
||||
}
|
||||
}
|
||||
|
||||
async function runContainerBackupJob(job) {
|
||||
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
||||
const snapshotName = `s3-backup-${timestamp}`;
|
||||
const dataset = `${config.zfsPoolName}/containers/${job.vmName}`;
|
||||
const snapshot = `${dataset}@snapshot-${snapshotName}`;
|
||||
|
||||
try {
|
||||
const totalBytes = await zfsDatasetUsed(dataset);
|
||||
|
||||
setJobRunning(job, 'Creating Incus snapshot');
|
||||
setJobProgress(job, { percent: 2 });
|
||||
await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
|
||||
|
||||
setJobStep(job, 'Streaming ZFS snapshot to Restic');
|
||||
const zfs = spawn('zfs', ['send', snapshot], { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
const zfsClosed = waitForProcess(zfs, 'zfs send');
|
||||
zfs.stderr.on('data', (chunk) => appendJobLog(job, chunk.toString().trimEnd()));
|
||||
const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
|
||||
setJobProgress(job, {
|
||||
currentBytes,
|
||||
totalBytes: bytesTotal,
|
||||
percent: 10 + percent * 0.78,
|
||||
detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
|
||||
});
|
||||
});
|
||||
let resticOk = false;
|
||||
try {
|
||||
await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'container'], {
|
||||
env: { ...process.env, ...config.resticEnv },
|
||||
input: zfs.stdout.pipe(progressStream),
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
resticOk = true;
|
||||
} finally {
|
||||
if (!resticOk && !zfs.killed) zfs.kill('SIGTERM');
|
||||
}
|
||||
await zfsClosed;
|
||||
|
||||
setJobStep(job, 'Deleting temporary Incus snapshot');
|
||||
setJobProgress(job, { percent: 93 });
|
||||
await deleteIncusSnapshotWithRetry(job, snapshotName);
|
||||
|
||||
setJobStep(job, 'Applying Restic retention policy');
|
||||
setJobProgress(job, { percent: 96 });
|
||||
await runRestic(retentionArgs(job.vmName), {
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
|
||||
finishJob(job, 'success');
|
||||
} catch (error) {
|
||||
appendJobLog(job, error.message);
|
||||
await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
|
||||
finishJob(job, 'failed', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function deviceReadStream(path) {
|
||||
const { createReadStream } = await import('node:fs');
|
||||
return createReadStream(path);
|
||||
@@ -103,6 +167,25 @@ async function zfsVolumeSize(zvol) {
|
||||
return Number.isFinite(size) && size > 0 ? size : null;
|
||||
}
|
||||
|
||||
async function zfsDatasetUsed(dataset) {
|
||||
const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'used', dataset]);
|
||||
const size = Number(result.stdout.trim());
|
||||
return Number.isFinite(size) && size > 0 ? size : null;
|
||||
}
|
||||
|
||||
function waitForProcess(child, label) {
|
||||
return new Promise((resolve, reject) => {
|
||||
child.on('error', reject);
|
||||
child.on('close', (exitCode) => {
|
||||
if (exitCode !== 0 && exitCode !== null) {
|
||||
reject(new Error(`${label} exited with ${exitCode}`));
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function formatBytes(value) {
|
||||
if (!value) return '-';
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||
|
||||
@@ -16,7 +16,12 @@ restoreRouter.post('/:vmName', async (req, res, next) => {
|
||||
throw error;
|
||||
}
|
||||
|
||||
await validateVmExists(vmName);
|
||||
const instance = await validateVmExists(vmName);
|
||||
if (instance.type === 'container') {
|
||||
const error = new Error('Container restore is not implemented yet. Container backups can be created, but restore needs a safe zfs receive workflow.');
|
||||
error.status = 501;
|
||||
throw error;
|
||||
}
|
||||
const snapshot = await validateSnapshotForVm(vmName, snapshotId);
|
||||
const job = createJob('restore', vmName);
|
||||
runRestoreJob(job, snapshot.id).catch(() => {});
|
||||
|
||||
@@ -12,6 +12,7 @@ vmsRouter.get('/', async (_req, res, next) => {
|
||||
const latestJob = latestJobForVm(vm.name);
|
||||
return {
|
||||
name: vm.name,
|
||||
type: vm.type,
|
||||
status: vm.status,
|
||||
activeJob: activeJob ? summarizeJob(activeJob) : null,
|
||||
lastJobStatus: latestJob?.status || null,
|
||||
|
||||
@@ -20,22 +20,26 @@ export function assertSnapshotIdShape(snapshotId) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function listIncusVms() {
|
||||
export async function listIncusInstances() {
|
||||
const result = await spawnCommand('incus', ['list', '--format', 'json']);
|
||||
const entries = JSON.parse(result.stdout || '[]');
|
||||
return entries.filter((entry) => entry.type === 'virtual-machine');
|
||||
return entries.filter((entry) => ['virtual-machine', 'container'].includes(entry.type));
|
||||
}
|
||||
|
||||
export async function validateVmExists(vmName) {
|
||||
assertVmNameShape(vmName);
|
||||
const vms = await listIncusVms();
|
||||
const vm = vms.find((entry) => entry.name === vmName);
|
||||
if (!vm) {
|
||||
const error = new Error(`VM "${vmName}" was not found.`);
|
||||
export async function listIncusVms() {
|
||||
return listIncusInstances();
|
||||
}
|
||||
|
||||
export async function validateVmExists(instanceName) {
|
||||
assertVmNameShape(instanceName);
|
||||
const instances = await listIncusInstances();
|
||||
const instance = instances.find((entry) => entry.name === instanceName);
|
||||
if (!instance) {
|
||||
const error = new Error(`Instance "${instanceName}" was not found.`);
|
||||
error.status = 404;
|
||||
throw error;
|
||||
}
|
||||
return vm;
|
||||
return instance;
|
||||
}
|
||||
|
||||
export async function listSnapshotsForVm(vmName) {
|
||||
|
||||
Reference in New Issue
Block a user