diff --git a/README.md b/README.md index 2d7445e..64f5f71 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Required commands: - `incus` - `zfs` +- `zpool` - `restic` - `udevadm` - `dd` diff --git a/backend/src/routes/health.js b/backend/src/routes/health.js index e70603d..bc4ed02 100644 --- a/backend/src/routes/health.js +++ b/backend/src/routes/health.js @@ -1,5 +1,7 @@ import { Router } from 'express'; -import { missingEnvVars } from '../config.js'; +import { access } from 'node:fs/promises'; +import { constants } from 'node:fs'; +import { config, missingEnvVars } from '../config.js'; import { spawnCommand, runRestic } from '../executor.js'; export const healthRouter = Router(); @@ -7,35 +9,73 @@ export const healthRouter = Router(); healthRouter.get('/', async (_req, res) => { const missing = missingEnvVars(); const checks = { - config: missing.length ? `missing: ${missing.join(', ')}` : 'ok', - incus: 'pending', - zfs: 'pending', - restic: 'pending', + config: checkValue(!missing.length, missing.length ? `missing: ${missing.join(', ')}` : 'ok'), + commands: await checkCommands(['incus', 'zfs', 'zpool', 'restic', 'udevadm', 'dd']), + zfsPool: await checkZfsPool(), + zvol: await checkZvolAccess(), + resticRepository: await checkResticRepository(), }; - await Promise.all([ - checkCommand('incus', ['version']).then((value) => { - checks.incus = value; - }), - checkCommand('zfs', ['version']).then((value) => { - checks.zfs = value; - }), - runRestic(['snapshots', '--json'], { ignoreExitCode: true }).then((result) => { - checks.restic = result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed'; - }).catch((error) => { - checks.restic = error.message; - }), - ]); - - const ok = Object.values(checks).every((value) => value === 'ok'); + const ok = Object.values(checks).every((value) => value.ok); res.status(ok ? 200 : 503).json({ ok, checks }); }); +async function checkCommands(commands) { + const results = {}; + await Promise.all(commands.map(async (command) => { + results[command] = await checkCommand(command, ['--version']); + })); + const failed = Object.entries(results).filter(([, result]) => !result.ok); + return { + ok: failed.length === 0, + message: failed.length ? `failed: ${failed.map(([command]) => command).join(', ')}` : 'ok', + details: results, + }; +} + async function checkCommand(command, args) { try { const result = await spawnCommand(command, args, { ignoreExitCode: true }); - return result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed'; + return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || result.stdout.trim() || 'failed'); } catch (error) { - return error.message; + return checkValue(false, error.message); } } + +async function checkZfsPool() { + if (!config.zfsPoolName) return checkValue(false, 'ZFS_POOL_NAME is not configured'); + try { + const result = await spawnCommand('zpool', ['list', '-Hp', '-o', 'name,size,alloc,free,cap', config.zfsPoolName], { ignoreExitCode: true }); + if (result.exitCode !== 0) return checkValue(false, result.stderr.trim() || 'zpool list failed'); + const [name, size, allocated, free, capacity] = result.stdout.trim().split('\t'); + return { + ok: true, + message: 'ok', + details: { name, size: Number(size), allocated: Number(allocated), free: Number(free), capacity }, + }; + } catch (error) { + return checkValue(false, error.message); + } +} + +async function checkZvolAccess() { + try { + await access('/dev/zvol', constants.R_OK); + return checkValue(true, 'ok'); + } catch (error) { + return checkValue(false, error.message); + } +} + +async function checkResticRepository() { + try { + const result = await runRestic(['snapshots', '--json'], { ignoreExitCode: true }); + return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'restic repository check failed'); + } catch (error) { + return checkValue(false, error.message); + } +} + +function checkValue(ok, message, details = null) { + return { ok, message, ...(details ? { details } : {}) }; +} diff --git a/docs/issues.md b/docs/issues.md index 4191e5a..8de722a 100644 --- a/docs/issues.md +++ b/docs/issues.md @@ -32,7 +32,7 @@ Make `/api/health` useful for diagnosing whether a node can actually run backup ### Tasks -- Check that required commands exist: `incus`, `zfs`, `restic`, `udevadm`, `dd`. +- Check that required commands exist: `incus`, `zfs`, `zpool`, `restic`, `udevadm`, `dd`. - Check that configured ZFS pool exists. - Check that `/dev/zvol` is accessible. - Check Restic repository access. diff --git a/frontend/src/components/Operations.jsx b/frontend/src/components/Operations.jsx index ac094db..3736700 100644 --- a/frontend/src/components/Operations.jsx +++ b/frontend/src/components/Operations.jsx @@ -31,13 +31,15 @@ export function Operations() { {error ?
{error}
: null} [ formatTime(row.startedAt), row.type, row.nodeName, row.vmName, row.error || row.status, + formatDuration(row.durationMs), + row.snapshotId || '-', row.agentJobId || '-', ])} title="Job History" @@ -86,3 +88,10 @@ function formatTime(value) { if (!value) return '-'; return new Intl.DateTimeFormat(undefined, { dateStyle: 'short', timeStyle: 'medium' }).format(new Date(value)); } + +function formatDuration(value) { + if (!value) return '-'; + const seconds = Math.round(Number(value) / 1000); + if (seconds < 60) return `${seconds}s`; + return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; +} diff --git a/management/src/db.js b/management/src/db.js index c964167..539c087 100644 --- a/management/src/db.js +++ b/management/src/db.js @@ -65,10 +65,18 @@ export function migrate() { agent_job_id TEXT, status TEXT NOT NULL, error TEXT, + current_step TEXT, + snapshot_id TEXT, started_at TEXT NOT NULL, + finished_at TEXT, + duration_ms INTEGER, updated_at TEXT NOT NULL ); `); + addColumnIfMissing('job_history', 'current_step', 'TEXT'); + addColumnIfMissing('job_history', 'snapshot_id', 'TEXT'); + addColumnIfMissing('job_history', 'finished_at', 'TEXT'); + addColumnIfMissing('job_history', 'duration_ms', 'INTEGER'); const existing = db.prepare('SELECT id FROM users LIMIT 1').get(); if (!existing) { @@ -82,3 +90,10 @@ export function migrate() { function cryptoId() { return Math.random().toString(16).slice(2) + Date.now().toString(16); } + +function addColumnIfMissing(table, column, definition) { + const columns = db.prepare(`PRAGMA table_info(${table})`).all().map((row) => row.name); + if (!columns.includes(column)) { + db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`); + } +} diff --git a/management/src/index.js b/management/src/index.js index 9e2dfc6..9210a26 100644 --- a/management/src/index.js +++ b/management/src/index.js @@ -3,6 +3,7 @@ import express from 'express'; import { config } from './config.js'; import { migrate } from './db.js'; import { requireAuth } from './auth.js'; +import { startJobPoller } from './jobPoller.js'; import { authRouter } from './routes/auth.js'; import { nodesRouter } from './routes/nodes.js'; import { operationsRouter } from './routes/operations.js'; @@ -33,3 +34,4 @@ app.listen(config.port, () => { }); startScheduler(); +startJobPoller(); diff --git a/management/src/jobPoller.js b/management/src/jobPoller.js new file mode 100644 index 0000000..c2d0a02 --- /dev/null +++ b/management/src/jobPoller.js @@ -0,0 +1,43 @@ +import { agentRequest } from './agentClient.js'; +import { listOpenJobHistory, updateJobHistory } from './store.js'; + +let timer = null; + +export function startJobPoller() { + if (timer) return; + timer = setInterval(pollJobs, 5000); + timer.unref?.(); + pollJobs(); +} + +async function pollJobs() { + const jobs = listOpenJobHistory(); + await Promise.all(jobs.map(async (history) => { + try { + const job = await agentRequest({ + id: history.node_id, + baseUrl: history.base_url, + token: history.token, + }, `/jobs/${encodeURIComponent(history.agent_job_id)}`, { timeout: 10000 }); + + updateJobHistory(history.id, { + status: job.status, + error: job.error, + currentStep: job.currentStep, + snapshotId: findSnapshotId(job), + finishedAt: job.finishedAt, + }); + } catch (error) { + updateJobHistory(history.id, { + status: 'failed', + error: error.message, + finishedAt: new Date().toISOString(), + }); + } + })); +} + +function findSnapshotId(job) { + const line = [...(job.logs || [])].reverse().find((entry) => /^snapshot [a-f0-9]+ saved$/i.test(entry)); + return line ? line.split(' ')[1] : null; +} diff --git a/management/src/store.js b/management/src/store.js index f3358f3..5f25ad4 100644 --- a/management/src/store.js +++ b/management/src/store.js @@ -108,8 +108,36 @@ export function createJobHistory({ node, vmName, type, agentJobId = null, status } export function updateJobHistory(id, values) { - db.prepare('UPDATE job_history SET status = ?, error = ?, updated_at = ? WHERE id = ?') - .run(values.status, values.error || null, new Date().toISOString(), id); + const current = db.prepare('SELECT * FROM job_history WHERE id = ?').get(id); + if (!current) return; + const finishedAt = values.finishedAt ?? current.finished_at; + const durationMs = finishedAt && current.started_at ? new Date(finishedAt).getTime() - new Date(current.started_at).getTime() : current.duration_ms; + db.prepare(` + UPDATE job_history + SET status = ?, error = ?, current_step = ?, snapshot_id = ?, finished_at = ?, duration_ms = ?, updated_at = ? + WHERE id = ? + `).run( + values.status ?? current.status, + values.error ?? current.error, + values.currentStep ?? current.current_step, + values.snapshotId ?? current.snapshot_id, + finishedAt, + Number.isFinite(durationMs) ? durationMs : null, + new Date().toISOString(), + id, + ); +} + +export function listOpenJobHistory() { + return db.prepare(` + SELECT job_history.*, nodes.base_url, nodes.token + FROM job_history + JOIN nodes ON nodes.id = job_history.node_id + WHERE job_history.agent_job_id IS NOT NULL + AND job_history.status IN ('accepted', 'queued', 'running') + AND nodes.enabled = 1 + ORDER BY job_history.started_at ASC + `).all(); } export function listJobHistory(limit = 100) { @@ -122,7 +150,11 @@ export function listJobHistory(limit = 100) { agentJobId: row.agent_job_id, status: row.status, error: row.error, + currentStep: row.current_step, + snapshotId: row.snapshot_id, startedAt: row.started_at, + finishedAt: row.finished_at, + durationMs: row.duration_ms, updatedAt: row.updated_at, })); }