added to remote repo

This commit is contained in:
Philipp
2026-05-21 08:18:29 +02:00
parent 8697c9f405
commit cc599b6e18
8049 changed files with 1096323 additions and 0 deletions
BIN
View File
Binary file not shown.
+106
View File
@@ -0,0 +1,106 @@
import dotenv from 'dotenv';
import { readFile, rename, writeFile } from 'node:fs/promises';
import path from 'node:path';
dotenv.config();
const envPath = path.resolve(process.cwd(), '.env');
export const requiredEnv = [
'AWS_ACCESS_KEY_ID',
'AWS_SECRET_ACCESS_KEY',
'RESTIC_REPOSITORY',
'RESTIC_PASSWORD',
'ZFS_POOL_NAME',
];
export const config = {
port: Number(process.env.PORT || 3000),
apiToken: process.env.API_TOKEN || '',
zfsPoolName: process.env.ZFS_POOL_NAME || '',
resticEnv: {
AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID || '',
AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY || '',
RESTIC_REPOSITORY: process.env.RESTIC_REPOSITORY || '',
RESTIC_PASSWORD: process.env.RESTIC_PASSWORD || '',
},
};
export const editableEnv = [
{ key: 'AWS_ACCESS_KEY_ID', label: 'AWS access key ID', required: true, secret: true },
{ key: 'AWS_SECRET_ACCESS_KEY', label: 'AWS secret access key', required: true, secret: true },
{ key: 'RESTIC_REPOSITORY', label: 'Restic repository', required: true, secret: false },
{ key: 'RESTIC_PASSWORD', label: 'Restic password', required: true, secret: true },
{ key: 'ZFS_POOL_NAME', label: 'ZFS pool name', required: true, secret: false },
{ key: 'PORT', label: 'API port', required: false, secret: false },
{ key: 'API_TOKEN', label: 'API token', required: false, secret: true },
];
export function missingEnvVars() {
return requiredEnv.filter((key) => !process.env[key]);
}
export function resticProcessEnv() {
return {
...process.env,
...config.resticEnv,
};
}
export async function readEnvSettings() {
const fileValues = await readEnvFile();
return editableEnv.map((field) => ({
...field,
value: fileValues[field.key] ?? process.env[field.key] ?? '',
}));
}
export async function writeEnvSettings(values) {
const allowedKeys = new Set(editableEnv.map((field) => field.key));
const currentValues = await readEnvFile();
const nextValues = { ...currentValues };
for (const [key, value] of Object.entries(values || {})) {
if (!allowedKeys.has(key)) continue;
nextValues[key] = String(value ?? '');
}
const body = editableEnv
.map(({ key }) => `${key}=${quoteEnvValue(nextValues[key] || '')}`)
.join('\n');
const tempPath = `${envPath}.tmp`;
await writeFile(tempPath, `${body}\n`, { mode: 0o600 });
await rename(tempPath, envPath);
applyRuntimeEnv(nextValues);
return readEnvSettings();
}
async function readEnvFile() {
try {
const content = await readFile(envPath, 'utf8');
return dotenv.parse(content);
} catch (error) {
if (error.code === 'ENOENT') return {};
throw error;
}
}
function quoteEnvValue(value) {
const escaped = String(value).replace(/\\/g, '\\\\').replace(/"/g, '\\"');
return `"${escaped}"`;
}
function applyRuntimeEnv(values) {
for (const [key, value] of Object.entries(values)) {
process.env[key] = value;
}
config.port = Number(process.env.PORT || 3000);
config.apiToken = process.env.API_TOKEN || '';
config.zfsPoolName = process.env.ZFS_POOL_NAME || '';
config.resticEnv.AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID || '';
config.resticEnv.AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY || '';
config.resticEnv.RESTIC_REPOSITORY = process.env.RESTIC_REPOSITORY || '';
config.resticEnv.RESTIC_PASSWORD = process.env.RESTIC_PASSWORD || '';
}
+127
View File
@@ -0,0 +1,127 @@
import { spawn } from 'node:child_process';
import { PassThrough } from 'node:stream';
import { resticProcessEnv } from './config.js';
export class CommandError extends Error {
constructor(message, result) {
super(message);
this.name = 'CommandError';
this.result = result;
}
}
export function spawnCommand(command, args = [], options = {}) {
const {
env = process.env,
input = null,
log = null,
ignoreExitCode = false,
cwd = process.cwd(),
} = options;
return new Promise((resolve, reject) => {
const child = spawn(command, args, {
cwd,
env,
shell: false,
stdio: ['pipe', 'pipe', 'pipe'],
});
let stdout = '';
let stderr = '';
child.stdout.on('data', (chunk) => {
const text = chunk.toString();
stdout += text;
log?.(text.trimEnd());
});
child.stderr.on('data', (chunk) => {
const text = chunk.toString();
stderr += text;
log?.(text.trimEnd());
});
child.on('error', (error) => {
reject(error);
});
child.on('close', (exitCode) => {
const result = { stdout, stderr, exitCode };
if (exitCode !== 0 && !ignoreExitCode) {
reject(new CommandError(stderr.trim() || `${command} exited with ${exitCode}`, result));
return;
}
resolve(result);
});
if (input) {
input.on('error', (error) => {
child.kill('SIGTERM');
reject(error);
});
input.pipe(child.stdin);
} else {
child.stdin.end();
}
});
}
export function runRestic(args, options = {}) {
return spawnCommand('restic', args, {
...options,
env: resticProcessEnv(),
});
}
export function streamResticDumpToDd(snapshotId, filename, outputPath, log) {
return new Promise((resolve, reject) => {
const restic = spawn('restic', ['dump', snapshotId, filename], {
env: resticProcessEnv(),
shell: false,
stdio: ['ignore', 'pipe', 'pipe'],
});
const dd = spawn('dd', [`of=${outputPath}`, 'bs=4M', 'conv=sparse', 'status=none'], {
env: process.env,
shell: false,
stdio: ['pipe', 'pipe', 'pipe'],
});
const errors = [];
let resticClosed = false;
let ddClosed = false;
const pipe = new PassThrough();
restic.stdout.pipe(pipe).pipe(dd.stdin);
restic.stderr.on('data', (chunk) => log?.(chunk.toString().trimEnd()));
dd.stderr.on('data', (chunk) => log?.(chunk.toString().trimEnd()));
restic.on('error', reject);
dd.on('error', reject);
restic.on('close', (code) => {
resticClosed = true;
if (code !== 0) {
errors.push(`restic dump exited with ${code}`);
dd.stdin.destroy();
}
maybeFinish();
});
dd.on('close', (code) => {
ddClosed = true;
if (code !== 0) {
errors.push(`dd exited with ${code}`);
}
maybeFinish();
});
function maybeFinish() {
if (!resticClosed || !ddClosed) return;
if (errors.length) {
reject(new Error(errors.join('; ')));
return;
}
resolve();
}
});
}
+45
View File
@@ -0,0 +1,45 @@
import cors from 'cors';
import express from 'express';
import { config } from './config.js';
import { backupRouter } from './routes/backup.js';
import { healthRouter } from './routes/health.js';
import { jobsRouter } from './routes/jobs.js';
import { restoreRouter } from './routes/restore.js';
import { settingsRouter } from './routes/settings.js';
import { snapshotsRouter } from './routes/snapshots.js';
import { vmsRouter } from './routes/vms.js';
const app = express();
app.use(cors());
app.use(express.json());
app.use((req, res, next) => {
if (!config.apiToken) {
next();
return;
}
const header = req.get('authorization') || '';
if (header === `Bearer ${config.apiToken}`) {
next();
return;
}
res.status(401).json({ error: 'Unauthorized.' });
});
app.use('/api/health', healthRouter);
app.use('/api/vms', vmsRouter);
app.use('/api/snapshots', snapshotsRouter);
app.use('/api/jobs', jobsRouter);
app.use('/api/backup', backupRouter);
app.use('/api/restore', restoreRouter);
app.use('/api/settings', settingsRouter);
app.use((error, _req, res, _next) => {
const status = error.status || 500;
res.status(status).json({ error: error.message || 'Internal server error.' });
});
app.listen(config.port, () => {
console.log(`Incus backup API listening on http://localhost:${config.port}`);
});
+85
View File
@@ -0,0 +1,85 @@
import crypto from 'node:crypto';
const jobs = new Map();
const locks = new Map();
const maxJobs = 100;
export function createJob(type, vmName) {
if (locks.has(vmName)) {
const activeJob = jobs.get(locks.get(vmName));
const error = new Error(`A ${activeJob?.type || 'job'} job is already active for ${vmName}.`);
error.status = 409;
throw error;
}
const id = `job_${crypto.randomBytes(8).toString('hex')}`;
const now = new Date().toISOString();
const job = {
id,
type,
vmName,
status: 'queued',
startedAt: now,
finishedAt: null,
currentStep: 'Queued',
logs: [],
error: null,
};
jobs.set(id, job);
locks.set(vmName, id);
trimJobs();
return job;
}
export function getJob(id) {
return jobs.get(id) || null;
}
export function listJobs() {
return [...jobs.values()].sort((a, b) => b.startedAt.localeCompare(a.startedAt));
}
export function activeJobForVm(vmName) {
const id = locks.get(vmName);
return id ? getJob(id) : null;
}
export function latestJobForVm(vmName) {
return listJobs().find((job) => job.vmName === vmName) || null;
}
export function setJobRunning(job, step) {
job.status = 'running';
setJobStep(job, step);
}
export function setJobStep(job, step) {
job.currentStep = step;
appendJobLog(job, `==> ${step}`);
}
export function appendJobLog(job, line) {
if (!line) return;
job.logs.push(...String(line).split('\n').filter(Boolean));
if (job.logs.length > 1000) {
job.logs = job.logs.slice(-1000);
}
}
export function finishJob(job, status, error = null) {
job.status = status;
job.finishedAt = new Date().toISOString();
job.currentStep = status === 'success' ? 'Completed' : 'Failed';
job.error = error ? String(error.message || error) : null;
locks.delete(job.vmName);
}
function trimJobs() {
const allJobs = listJobs();
for (const job of allJobs.slice(maxJobs)) {
if (job.status !== 'running' && job.status !== 'queued') {
jobs.delete(job.id);
}
}
}
+78
View File
@@ -0,0 +1,78 @@
import { Router } from 'express';
import { config } from '../config.js';
import { spawnCommand, runRestic } from '../executor.js';
import { appendJobLog, createJob, finishJob, setJobRunning, setJobStep } from '../jobs.js';
import { validateVmExists } from '../validators.js';
export const backupRouter = Router();
backupRouter.post('/:vmName', async (req, res, next) => {
try {
const vmName = req.params.vmName;
await validateVmExists(vmName);
const job = createJob('backup', vmName);
runBackupJob(job).catch(() => {});
res.status(202).json({ jobId: job.id, message: 'Backup job started.' });
} catch (error) {
next(error);
}
});
async function runBackupJob(job) {
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
const snapshotName = `s3-backup-${timestamp}`;
const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
const snapshotDevice = `/dev/zvol/${zvol}@snapshot-${snapshotName}`;
try {
setJobRunning(job, 'Creating Incus snapshot');
await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
setJobStep(job, 'Making ZFS snapshot device visible');
await spawnCommand('zfs', ['set', 'snapdev=visible', zvol], { log: (line) => appendJobLog(job, line) });
setJobStep(job, 'Waiting for snapshot device');
await new Promise((resolve) => setTimeout(resolve, 2000));
setJobStep(job, 'Streaming block device to Restic');
await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName], {
env: { ...process.env, ...config.resticEnv },
input: await deviceReadStream(snapshotDevice),
log: (line) => appendJobLog(job, line),
});
setJobStep(job, 'Hiding ZFS snapshot device');
await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { log: (line) => appendJobLog(job, line) });
setJobStep(job, 'Deleting temporary Incus snapshot');
await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
setJobStep(job, 'Applying Restic retention policy');
await runRestic(['forget', '--tag', job.vmName, '--keep-daily', '7', '--prune'], {
log: (line) => appendJobLog(job, line),
});
finishJob(job, 'success');
} catch (error) {
appendJobLog(job, error.message);
await cleanupBackup(job, zvol, snapshotName);
finishJob(job, 'failed', error);
}
}
async function deviceReadStream(path) {
const { createReadStream } = await import('node:fs');
return createReadStream(path);
}
async function cleanupBackup(job, zvol, snapshotName) {
setJobStep(job, 'Running cleanup');
await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], {
ignoreExitCode: true,
log: (line) => appendJobLog(job, line),
}).catch((error) => appendJobLog(job, error.message));
await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], {
ignoreExitCode: true,
log: (line) => appendJobLog(job, line),
}).catch((error) => appendJobLog(job, error.message));
}
+41
View File
@@ -0,0 +1,41 @@
import { Router } from 'express';
import { missingEnvVars } from '../config.js';
import { spawnCommand, runRestic } from '../executor.js';
export const healthRouter = Router();
healthRouter.get('/', async (_req, res) => {
const missing = missingEnvVars();
const checks = {
config: missing.length ? `missing: ${missing.join(', ')}` : 'ok',
incus: 'pending',
zfs: 'pending',
restic: 'pending',
};
await Promise.all([
checkCommand('incus', ['version']).then((value) => {
checks.incus = value;
}),
checkCommand('zfs', ['version']).then((value) => {
checks.zfs = value;
}),
runRestic(['snapshots', '--json'], { ignoreExitCode: true }).then((result) => {
checks.restic = result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed';
}).catch((error) => {
checks.restic = error.message;
}),
]);
const ok = Object.values(checks).every((value) => value === 'ok');
res.status(ok ? 200 : 503).json({ ok, checks });
});
async function checkCommand(command, args) {
try {
const result = await spawnCommand(command, args, { ignoreExitCode: true });
return result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed';
} catch (error) {
return error.message;
}
}
+17
View File
@@ -0,0 +1,17 @@
import { Router } from 'express';
import { getJob, listJobs } from '../jobs.js';
export const jobsRouter = Router();
jobsRouter.get('/', (_req, res) => {
res.json(listJobs());
});
jobsRouter.get('/:jobId', (req, res) => {
const job = getJob(req.params.jobId);
if (!job) {
res.status(404).json({ error: 'Job not found.' });
return;
}
res.json(job);
});
+77
View File
@@ -0,0 +1,77 @@
import { Router } from 'express';
import { config } from '../config.js';
import { spawnCommand, streamResticDumpToDd } from '../executor.js';
import { appendJobLog, createJob, finishJob, setJobRunning, setJobStep } from '../jobs.js';
import { validateSnapshotForVm, validateVmExists } from '../validators.js';
export const restoreRouter = Router();
restoreRouter.post('/:vmName', async (req, res, next) => {
try {
const vmName = req.params.vmName;
const { snapshotId, confirmVmName } = req.body || {};
if (confirmVmName !== vmName) {
const error = new Error('Restore confirmation does not match VM name.');
error.status = 400;
throw error;
}
await validateVmExists(vmName);
const snapshot = await validateSnapshotForVm(vmName, snapshotId);
const job = createJob('restore', vmName);
runRestoreJob(job, snapshot.id).catch(() => {});
res.status(202).json({ jobId: job.id, message: 'Restore job started.' });
} catch (error) {
next(error);
}
});
async function runRestoreJob(job, snapshotId) {
const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
const device = `/dev/zvol/${zvol}`;
let volmodeDev = false;
let diskWriteOk = false;
try {
setJobRunning(job, 'Stopping VM');
await spawnCommand('incus', ['stop', job.vmName, '--force'], {
ignoreExitCode: true,
log: (line) => appendJobLog(job, line),
});
setJobStep(job, 'Setting ZFS volume to device mode');
await spawnCommand('zfs', ['set', 'volmode=dev', zvol], { log: (line) => appendJobLog(job, line) });
volmodeDev = true;
setJobStep(job, 'Settling device nodes');
await spawnCommand('udevadm', ['trigger'], { log: (line) => appendJobLog(job, line) });
await spawnCommand('udevadm', ['settle'], { log: (line) => appendJobLog(job, line) });
await new Promise((resolve) => setTimeout(resolve, 2000));
setJobStep(job, 'Writing Restic snapshot to block device');
await streamResticDumpToDd(snapshotId, `${job.vmName}.raw`, device, (line) => appendJobLog(job, line));
diskWriteOk = true;
setJobStep(job, 'Restoring ZFS volume mode');
await spawnCommand('zfs', ['set', 'volmode=none', zvol], { log: (line) => appendJobLog(job, line) });
volmodeDev = false;
setJobStep(job, 'Starting VM');
await spawnCommand('incus', ['start', job.vmName], { log: (line) => appendJobLog(job, line) });
finishJob(job, 'success');
} catch (error) {
appendJobLog(job, error.message);
if (volmodeDev) {
setJobStep(job, 'Restoring ZFS volume mode after failure');
await spawnCommand('zfs', ['set', 'volmode=none', zvol], {
ignoreExitCode: true,
log: (line) => appendJobLog(job, line),
}).catch((cleanupError) => appendJobLog(job, cleanupError.message));
}
if (!diskWriteOk) {
appendJobLog(job, 'VM was not restarted because disk restore did not complete successfully.');
}
finishJob(job, 'failed', error);
}
}
+23
View File
@@ -0,0 +1,23 @@
import { Router } from 'express';
import { readEnvSettings, writeEnvSettings } from '../config.js';
export const settingsRouter = Router();
settingsRouter.get('/', async (_req, res, next) => {
try {
res.json({ fields: await readEnvSettings() });
} catch (error) {
next(error);
}
});
settingsRouter.put('/', async (req, res, next) => {
try {
res.json({
fields: await writeEnvSettings(req.body?.values || {}),
message: 'Settings saved.',
});
} catch (error) {
next(error);
}
});
+12
View File
@@ -0,0 +1,12 @@
import { Router } from 'express';
import { listSnapshotsForVm } from '../validators.js';
export const snapshotsRouter = Router();
snapshotsRouter.get('/:vmName', async (req, res, next) => {
try {
res.json(await listSnapshotsForVm(req.params.vmName));
} catch (error) {
next(error);
}
});
+32
View File
@@ -0,0 +1,32 @@
import { Router } from 'express';
import { activeJobForVm, latestJobForVm } from '../jobs.js';
import { listIncusVms } from '../validators.js';
export const vmsRouter = Router();
vmsRouter.get('/', async (_req, res, next) => {
try {
const vms = await listIncusVms();
res.json(vms.map((vm) => {
const activeJob = activeJobForVm(vm.name);
const latestJob = latestJobForVm(vm.name);
return {
name: vm.name,
status: vm.status,
activeJob: activeJob ? summarizeJob(activeJob) : null,
lastJobStatus: latestJob?.status || null,
};
}));
} catch (error) {
next(error);
}
});
function summarizeJob(job) {
return {
id: job.id,
type: job.type,
status: job.status,
currentStep: job.currentStep,
};
}
+67
View File
@@ -0,0 +1,67 @@
import { spawnCommand, runRestic } from './executor.js';
const vmNamePattern = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/;
const snapshotIdPattern = /^[A-Fa-f0-9]{8,64}$/;
export function assertVmNameShape(vmName) {
if (!vmNamePattern.test(vmName || '')) {
const error = new Error('Invalid VM name.');
error.status = 400;
throw error;
}
}
export function assertSnapshotIdShape(snapshotId) {
if (snapshotId === 'latest') return;
if (!snapshotIdPattern.test(snapshotId || '')) {
const error = new Error('Invalid snapshot id.');
error.status = 400;
throw error;
}
}
export async function listIncusVms() {
const result = await spawnCommand('incus', ['list', '--format', 'json']);
const entries = JSON.parse(result.stdout || '[]');
return entries.filter((entry) => entry.type === 'virtual-machine');
}
export async function validateVmExists(vmName) {
assertVmNameShape(vmName);
const vms = await listIncusVms();
const vm = vms.find((entry) => entry.name === vmName);
if (!vm) {
const error = new Error(`VM "${vmName}" was not found.`);
error.status = 404;
throw error;
}
return vm;
}
export async function listSnapshotsForVm(vmName) {
await validateVmExists(vmName);
const result = await runRestic(['snapshots', '--json', '--tag', vmName]);
const snapshots = JSON.parse(result.stdout || '[]');
return snapshots.sort((a, b) => String(b.time || '').localeCompare(String(a.time || '')));
}
export async function validateSnapshotForVm(vmName, snapshotId) {
assertSnapshotIdShape(snapshotId);
const snapshots = await listSnapshotsForVm(vmName);
if (snapshotId === 'latest') {
if (!snapshots.length) {
const error = new Error(`No snapshots found for ${vmName}.`);
error.status = 404;
throw error;
}
return snapshots[0];
}
const snapshot = snapshots.find((entry) => entry.id?.startsWith(snapshotId));
if (!snapshot) {
const error = new Error(`Snapshot "${snapshotId}" was not found for ${vmName}.`);
error.status = 404;
throw error;
}
return snapshot;
}