added support for container Backup
This commit is contained in:
@@ -1,4 +1,5 @@
|
|||||||
import { Router } from 'express';
|
import { Router } from 'express';
|
||||||
|
import { spawn } from 'node:child_process';
|
||||||
import { finished } from 'node:stream/promises';
|
import { finished } from 'node:stream/promises';
|
||||||
import { setTimeout as delay } from 'node:timers/promises';
|
import { setTimeout as delay } from 'node:timers/promises';
|
||||||
import { config } from '../config.js';
|
import { config } from '../config.js';
|
||||||
@@ -19,13 +20,20 @@ backupRouter.post('/:vmName', async (req, res, next) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
export async function startBackupForVm(vmName) {
|
export async function startBackupForVm(vmName) {
|
||||||
await validateVmExists(vmName);
|
const instance = await validateVmExists(vmName);
|
||||||
const job = createJob('backup', vmName);
|
const job = createJob('backup', vmName);
|
||||||
runBackupJob(job).catch(() => {});
|
job.instanceType = instance.type;
|
||||||
|
runBackupJob(job, instance).catch(() => {});
|
||||||
return job;
|
return job;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function runBackupJob(job) {
|
export async function runBackupJob(job, instance = null) {
|
||||||
|
const currentInstance = instance || await validateVmExists(job.vmName);
|
||||||
|
if (currentInstance.type === 'container') {
|
||||||
|
await runContainerBackupJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
||||||
const snapshotName = `s3-backup-${timestamp}`;
|
const snapshotName = `s3-backup-${timestamp}`;
|
||||||
const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
|
const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
|
||||||
@@ -92,6 +100,62 @@ export async function runBackupJob(job) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function runContainerBackupJob(job) {
|
||||||
|
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
||||||
|
const snapshotName = `s3-backup-${timestamp}`;
|
||||||
|
const dataset = `${config.zfsPoolName}/containers/${job.vmName}`;
|
||||||
|
const snapshot = `${dataset}@snapshot-${snapshotName}`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const totalBytes = await zfsDatasetUsed(dataset);
|
||||||
|
|
||||||
|
setJobRunning(job, 'Creating Incus snapshot');
|
||||||
|
setJobProgress(job, { percent: 2 });
|
||||||
|
await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
|
||||||
|
|
||||||
|
setJobStep(job, 'Streaming ZFS snapshot to Restic');
|
||||||
|
const zfs = spawn('zfs', ['send', snapshot], { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||||
|
const zfsClosed = waitForProcess(zfs, 'zfs send');
|
||||||
|
zfs.stderr.on('data', (chunk) => appendJobLog(job, chunk.toString().trimEnd()));
|
||||||
|
const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
|
||||||
|
setJobProgress(job, {
|
||||||
|
currentBytes,
|
||||||
|
totalBytes: bytesTotal,
|
||||||
|
percent: 10 + percent * 0.78,
|
||||||
|
detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
let resticOk = false;
|
||||||
|
try {
|
||||||
|
await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'container'], {
|
||||||
|
env: { ...process.env, ...config.resticEnv },
|
||||||
|
input: zfs.stdout.pipe(progressStream),
|
||||||
|
log: (line) => appendJobLog(job, line),
|
||||||
|
});
|
||||||
|
resticOk = true;
|
||||||
|
} finally {
|
||||||
|
if (!resticOk && !zfs.killed) zfs.kill('SIGTERM');
|
||||||
|
}
|
||||||
|
await zfsClosed;
|
||||||
|
|
||||||
|
setJobStep(job, 'Deleting temporary Incus snapshot');
|
||||||
|
setJobProgress(job, { percent: 93 });
|
||||||
|
await deleteIncusSnapshotWithRetry(job, snapshotName);
|
||||||
|
|
||||||
|
setJobStep(job, 'Applying Restic retention policy');
|
||||||
|
setJobProgress(job, { percent: 96 });
|
||||||
|
await runRestic(retentionArgs(job.vmName), {
|
||||||
|
log: (line) => appendJobLog(job, line),
|
||||||
|
});
|
||||||
|
|
||||||
|
finishJob(job, 'success');
|
||||||
|
} catch (error) {
|
||||||
|
appendJobLog(job, error.message);
|
||||||
|
await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
|
||||||
|
finishJob(job, 'failed', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function deviceReadStream(path) {
|
async function deviceReadStream(path) {
|
||||||
const { createReadStream } = await import('node:fs');
|
const { createReadStream } = await import('node:fs');
|
||||||
return createReadStream(path);
|
return createReadStream(path);
|
||||||
@@ -103,6 +167,25 @@ async function zfsVolumeSize(zvol) {
|
|||||||
return Number.isFinite(size) && size > 0 ? size : null;
|
return Number.isFinite(size) && size > 0 ? size : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function zfsDatasetUsed(dataset) {
|
||||||
|
const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'used', dataset]);
|
||||||
|
const size = Number(result.stdout.trim());
|
||||||
|
return Number.isFinite(size) && size > 0 ? size : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function waitForProcess(child, label) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
child.on('error', reject);
|
||||||
|
child.on('close', (exitCode) => {
|
||||||
|
if (exitCode !== 0 && exitCode !== null) {
|
||||||
|
reject(new Error(`${label} exited with ${exitCode}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
function formatBytes(value) {
|
function formatBytes(value) {
|
||||||
if (!value) return '-';
|
if (!value) return '-';
|
||||||
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||||
|
|||||||
@@ -16,7 +16,12 @@ restoreRouter.post('/:vmName', async (req, res, next) => {
|
|||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
|
||||||
await validateVmExists(vmName);
|
const instance = await validateVmExists(vmName);
|
||||||
|
if (instance.type === 'container') {
|
||||||
|
const error = new Error('Container restore is not implemented yet. Container backups can be created, but restore needs a safe zfs receive workflow.');
|
||||||
|
error.status = 501;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
const snapshot = await validateSnapshotForVm(vmName, snapshotId);
|
const snapshot = await validateSnapshotForVm(vmName, snapshotId);
|
||||||
const job = createJob('restore', vmName);
|
const job = createJob('restore', vmName);
|
||||||
runRestoreJob(job, snapshot.id).catch(() => {});
|
runRestoreJob(job, snapshot.id).catch(() => {});
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ vmsRouter.get('/', async (_req, res, next) => {
|
|||||||
const latestJob = latestJobForVm(vm.name);
|
const latestJob = latestJobForVm(vm.name);
|
||||||
return {
|
return {
|
||||||
name: vm.name,
|
name: vm.name,
|
||||||
|
type: vm.type,
|
||||||
status: vm.status,
|
status: vm.status,
|
||||||
activeJob: activeJob ? summarizeJob(activeJob) : null,
|
activeJob: activeJob ? summarizeJob(activeJob) : null,
|
||||||
lastJobStatus: latestJob?.status || null,
|
lastJobStatus: latestJob?.status || null,
|
||||||
|
|||||||
@@ -20,22 +20,26 @@ export function assertSnapshotIdShape(snapshotId) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function listIncusVms() {
|
export async function listIncusInstances() {
|
||||||
const result = await spawnCommand('incus', ['list', '--format', 'json']);
|
const result = await spawnCommand('incus', ['list', '--format', 'json']);
|
||||||
const entries = JSON.parse(result.stdout || '[]');
|
const entries = JSON.parse(result.stdout || '[]');
|
||||||
return entries.filter((entry) => entry.type === 'virtual-machine');
|
return entries.filter((entry) => ['virtual-machine', 'container'].includes(entry.type));
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function validateVmExists(vmName) {
|
export async function listIncusVms() {
|
||||||
assertVmNameShape(vmName);
|
return listIncusInstances();
|
||||||
const vms = await listIncusVms();
|
}
|
||||||
const vm = vms.find((entry) => entry.name === vmName);
|
|
||||||
if (!vm) {
|
export async function validateVmExists(instanceName) {
|
||||||
const error = new Error(`VM "${vmName}" was not found.`);
|
assertVmNameShape(instanceName);
|
||||||
|
const instances = await listIncusInstances();
|
||||||
|
const instance = instances.find((entry) => entry.name === instanceName);
|
||||||
|
if (!instance) {
|
||||||
|
const error = new Error(`Instance "${instanceName}" was not found.`);
|
||||||
error.status = 404;
|
error.status = 404;
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
return vm;
|
return instance;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function listSnapshotsForVm(vmName) {
|
export async function listSnapshotsForVm(vmName) {
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ export function Dashboard({ health, jobs, loading, onManage, vms }) {
|
|||||||
<tr>
|
<tr>
|
||||||
<th className="px-4 py-3 font-medium">VM</th>
|
<th className="px-4 py-3 font-medium">VM</th>
|
||||||
<th className="px-4 py-3 font-medium">Node</th>
|
<th className="px-4 py-3 font-medium">Node</th>
|
||||||
|
<th className="px-4 py-3 font-medium">Type</th>
|
||||||
<th className="px-4 py-3 font-medium">Incus</th>
|
<th className="px-4 py-3 font-medium">Incus</th>
|
||||||
<th className="px-4 py-3 font-medium">Last Job</th>
|
<th className="px-4 py-3 font-medium">Last Job</th>
|
||||||
<th className="px-4 py-3 font-medium">Active Job</th>
|
<th className="px-4 py-3 font-medium">Active Job</th>
|
||||||
@@ -38,6 +39,7 @@ export function Dashboard({ health, jobs, loading, onManage, vms }) {
|
|||||||
<tr className="hover:bg-zinc-900" key={vm.id || vm.name}>
|
<tr className="hover:bg-zinc-900" key={vm.id || vm.name}>
|
||||||
<td className="px-4 py-3 font-medium text-zinc-100">{vm.name}</td>
|
<td className="px-4 py-3 font-medium text-zinc-100">{vm.name}</td>
|
||||||
<td className="px-4 py-3 text-zinc-300">{vm.nodeName || '-'}</td>
|
<td className="px-4 py-3 text-zinc-300">{vm.nodeName || '-'}</td>
|
||||||
|
<td className="px-4 py-3 text-zinc-300">{vm.type === 'container' ? 'Container' : 'VM'}</td>
|
||||||
<td className="px-4 py-3">
|
<td className="px-4 py-3">
|
||||||
<span className="inline-flex items-center gap-2 text-zinc-300">
|
<span className="inline-flex items-center gap-2 text-zinc-300">
|
||||||
<Circle className={`h-2.5 w-2.5 fill-current ${vm.status === 'Running' ? 'text-emerald-400' : 'text-red-400'}`} />
|
<Circle className={`h-2.5 w-2.5 fill-current ${vm.status === 'Running' ? 'text-emerald-400' : 'text-red-400'}`} />
|
||||||
@@ -71,7 +73,7 @@ export function Dashboard({ health, jobs, loading, onManage, vms }) {
|
|||||||
))}
|
))}
|
||||||
{!vms.length ? (
|
{!vms.length ? (
|
||||||
<tr>
|
<tr>
|
||||||
<td className="px-4 py-8 text-center text-zinc-500" colSpan="6">
|
<td className="px-4 py-8 text-center text-zinc-500" colSpan="7">
|
||||||
No VMs loaded.
|
No VMs loaded.
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
|||||||
@@ -90,6 +90,7 @@ export function VMDetail({ jobs, onBack, onChanged, vm }) {
|
|||||||
</button>
|
</button>
|
||||||
<div className="flex items-center gap-3">
|
<div className="flex items-center gap-3">
|
||||||
<h1 className="text-2xl font-semibold text-zinc-100">{vm.name}</h1>
|
<h1 className="text-2xl font-semibold text-zinc-100">{vm.name}</h1>
|
||||||
|
<span className="rounded-md border border-zinc-800 px-2 py-1 text-xs text-zinc-300">{vm.type === 'container' ? 'Container' : 'VM'}</span>
|
||||||
<span className="rounded-md border border-zinc-800 px-2 py-1 text-xs text-zinc-300">{vm.status || 'Unknown'}</span>
|
<span className="rounded-md border border-zinc-800 px-2 py-1 text-xs text-zinc-300">{vm.status || 'Unknown'}</span>
|
||||||
</div>
|
</div>
|
||||||
<p className="mt-2 text-sm text-zinc-500">Node: {vm.nodeName || '-'}</p>
|
<p className="mt-2 text-sm text-zinc-500">Node: {vm.nodeName || '-'}</p>
|
||||||
@@ -118,7 +119,13 @@ export function VMDetail({ jobs, onBack, onChanged, vm }) {
|
|||||||
<SnapshotTable
|
<SnapshotTable
|
||||||
snapshots={snapshots}
|
snapshots={snapshots}
|
||||||
onInspect={inspectSnapshot}
|
onInspect={inspectSnapshot}
|
||||||
onRestore={(snapshot) => setSelectedSnapshot(snapshot)}
|
onRestore={(snapshot) => {
|
||||||
|
if (vm.type === 'container') {
|
||||||
|
setError('Container restore is not implemented yet. Backups are available, restore needs a safe zfs receive workflow.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setSelectedSnapshot(snapshot);
|
||||||
|
}}
|
||||||
/>
|
/>
|
||||||
<SnapshotFilesPanel
|
<SnapshotFilesPanel
|
||||||
files={snapshotFiles}
|
files={snapshotFiles}
|
||||||
|
|||||||
Reference in New Issue
Block a user