diff --git a/README.md b/README.md
index 2d7445e..64f5f71 100644
--- a/README.md
+++ b/README.md
@@ -26,6 +26,7 @@ Required commands:
- `incus`
- `zfs`
+- `zpool`
- `restic`
- `udevadm`
- `dd`
diff --git a/backend/src/routes/health.js b/backend/src/routes/health.js
index e70603d..bc4ed02 100644
--- a/backend/src/routes/health.js
+++ b/backend/src/routes/health.js
@@ -1,5 +1,7 @@
import { Router } from 'express';
-import { missingEnvVars } from '../config.js';
+import { access } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { config, missingEnvVars } from '../config.js';
import { spawnCommand, runRestic } from '../executor.js';
export const healthRouter = Router();
@@ -7,35 +9,73 @@ export const healthRouter = Router();
healthRouter.get('/', async (_req, res) => {
const missing = missingEnvVars();
const checks = {
- config: missing.length ? `missing: ${missing.join(', ')}` : 'ok',
- incus: 'pending',
- zfs: 'pending',
- restic: 'pending',
+ config: checkValue(!missing.length, missing.length ? `missing: ${missing.join(', ')}` : 'ok'),
+ commands: await checkCommands(['incus', 'zfs', 'zpool', 'restic', 'udevadm', 'dd']),
+ zfsPool: await checkZfsPool(),
+ zvol: await checkZvolAccess(),
+ resticRepository: await checkResticRepository(),
};
- await Promise.all([
- checkCommand('incus', ['version']).then((value) => {
- checks.incus = value;
- }),
- checkCommand('zfs', ['version']).then((value) => {
- checks.zfs = value;
- }),
- runRestic(['snapshots', '--json'], { ignoreExitCode: true }).then((result) => {
- checks.restic = result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed';
- }).catch((error) => {
- checks.restic = error.message;
- }),
- ]);
-
- const ok = Object.values(checks).every((value) => value === 'ok');
+ const ok = Object.values(checks).every((value) => value.ok);
res.status(ok ? 200 : 503).json({ ok, checks });
});
+async function checkCommands(commands) {
+ const results = {};
+ await Promise.all(commands.map(async (command) => {
+ results[command] = await checkCommand(command, ['--version']);
+ }));
+ const failed = Object.entries(results).filter(([, result]) => !result.ok);
+ return {
+ ok: failed.length === 0,
+ message: failed.length ? `failed: ${failed.map(([command]) => command).join(', ')}` : 'ok',
+ details: results,
+ };
+}
+
async function checkCommand(command, args) {
try {
const result = await spawnCommand(command, args, { ignoreExitCode: true });
- return result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed';
+ return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || result.stdout.trim() || 'failed');
} catch (error) {
- return error.message;
+ return checkValue(false, error.message);
}
}
+
+async function checkZfsPool() {
+ if (!config.zfsPoolName) return checkValue(false, 'ZFS_POOL_NAME is not configured');
+ try {
+ const result = await spawnCommand('zpool', ['list', '-Hp', '-o', 'name,size,alloc,free,cap', config.zfsPoolName], { ignoreExitCode: true });
+ if (result.exitCode !== 0) return checkValue(false, result.stderr.trim() || 'zpool list failed');
+ const [name, size, allocated, free, capacity] = result.stdout.trim().split('\t');
+ return {
+ ok: true,
+ message: 'ok',
+ details: { name, size: Number(size), allocated: Number(allocated), free: Number(free), capacity },
+ };
+ } catch (error) {
+ return checkValue(false, error.message);
+ }
+}
+
+async function checkZvolAccess() {
+ try {
+ await access('/dev/zvol', constants.R_OK);
+ return checkValue(true, 'ok');
+ } catch (error) {
+ return checkValue(false, error.message);
+ }
+}
+
+async function checkResticRepository() {
+ try {
+ const result = await runRestic(['snapshots', '--json'], { ignoreExitCode: true });
+ return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'restic repository check failed');
+ } catch (error) {
+ return checkValue(false, error.message);
+ }
+}
+
+function checkValue(ok, message, details = null) {
+ return { ok, message, ...(details ? { details } : {}) };
+}
diff --git a/docs/issues.md b/docs/issues.md
index 4191e5a..8de722a 100644
--- a/docs/issues.md
+++ b/docs/issues.md
@@ -32,7 +32,7 @@ Make `/api/health` useful for diagnosing whether a node can actually run backup
### Tasks
-- Check that required commands exist: `incus`, `zfs`, `restic`, `udevadm`, `dd`.
+- Check that required commands exist: `incus`, `zfs`, `zpool`, `restic`, `udevadm`, `dd`.
- Check that configured ZFS pool exists.
- Check that `/dev/zvol` is accessible.
- Check Restic repository access.
diff --git a/frontend/src/components/Operations.jsx b/frontend/src/components/Operations.jsx
index ac094db..3736700 100644
--- a/frontend/src/components/Operations.jsx
+++ b/frontend/src/components/Operations.jsx
@@ -31,13 +31,15 @@ export function Operations() {
{error ?
{error}
: null}
[
formatTime(row.startedAt),
row.type,
row.nodeName,
row.vmName,
row.error || row.status,
+ formatDuration(row.durationMs),
+ row.snapshotId || '-',
row.agentJobId || '-',
])}
title="Job History"
@@ -86,3 +88,10 @@ function formatTime(value) {
if (!value) return '-';
return new Intl.DateTimeFormat(undefined, { dateStyle: 'short', timeStyle: 'medium' }).format(new Date(value));
}
+
+function formatDuration(value) {
+ if (!value) return '-';
+ const seconds = Math.round(Number(value) / 1000);
+ if (seconds < 60) return `${seconds}s`;
+ return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
+}
diff --git a/management/src/db.js b/management/src/db.js
index c964167..539c087 100644
--- a/management/src/db.js
+++ b/management/src/db.js
@@ -65,10 +65,18 @@ export function migrate() {
agent_job_id TEXT,
status TEXT NOT NULL,
error TEXT,
+ current_step TEXT,
+ snapshot_id TEXT,
started_at TEXT NOT NULL,
+ finished_at TEXT,
+ duration_ms INTEGER,
updated_at TEXT NOT NULL
);
`);
+ addColumnIfMissing('job_history', 'current_step', 'TEXT');
+ addColumnIfMissing('job_history', 'snapshot_id', 'TEXT');
+ addColumnIfMissing('job_history', 'finished_at', 'TEXT');
+ addColumnIfMissing('job_history', 'duration_ms', 'INTEGER');
const existing = db.prepare('SELECT id FROM users LIMIT 1').get();
if (!existing) {
@@ -82,3 +90,10 @@ export function migrate() {
function cryptoId() {
return Math.random().toString(16).slice(2) + Date.now().toString(16);
}
+
+function addColumnIfMissing(table, column, definition) {
+ const columns = db.prepare(`PRAGMA table_info(${table})`).all().map((row) => row.name);
+ if (!columns.includes(column)) {
+ db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
+ }
+}
diff --git a/management/src/index.js b/management/src/index.js
index 9e2dfc6..9210a26 100644
--- a/management/src/index.js
+++ b/management/src/index.js
@@ -3,6 +3,7 @@ import express from 'express';
import { config } from './config.js';
import { migrate } from './db.js';
import { requireAuth } from './auth.js';
+import { startJobPoller } from './jobPoller.js';
import { authRouter } from './routes/auth.js';
import { nodesRouter } from './routes/nodes.js';
import { operationsRouter } from './routes/operations.js';
@@ -33,3 +34,4 @@ app.listen(config.port, () => {
});
startScheduler();
+startJobPoller();
diff --git a/management/src/jobPoller.js b/management/src/jobPoller.js
new file mode 100644
index 0000000..c2d0a02
--- /dev/null
+++ b/management/src/jobPoller.js
@@ -0,0 +1,43 @@
+import { agentRequest } from './agentClient.js';
+import { listOpenJobHistory, updateJobHistory } from './store.js';
+
+let timer = null;
+
+export function startJobPoller() {
+ if (timer) return;
+ timer = setInterval(pollJobs, 5000);
+ timer.unref?.();
+ pollJobs();
+}
+
+async function pollJobs() {
+ const jobs = listOpenJobHistory();
+ await Promise.all(jobs.map(async (history) => {
+ try {
+ const job = await agentRequest({
+ id: history.node_id,
+ baseUrl: history.base_url,
+ token: history.token,
+ }, `/jobs/${encodeURIComponent(history.agent_job_id)}`, { timeout: 10000 });
+
+ updateJobHistory(history.id, {
+ status: job.status,
+ error: job.error,
+ currentStep: job.currentStep,
+ snapshotId: findSnapshotId(job),
+ finishedAt: job.finishedAt,
+ });
+ } catch (error) {
+ updateJobHistory(history.id, {
+ status: 'failed',
+ error: error.message,
+ finishedAt: new Date().toISOString(),
+ });
+ }
+ }));
+}
+
+function findSnapshotId(job) {
+ const line = [...(job.logs || [])].reverse().find((entry) => /^snapshot [a-f0-9]+ saved$/i.test(entry));
+ return line ? line.split(' ')[1] : null;
+}
diff --git a/management/src/store.js b/management/src/store.js
index f3358f3..5f25ad4 100644
--- a/management/src/store.js
+++ b/management/src/store.js
@@ -108,8 +108,36 @@ export function createJobHistory({ node, vmName, type, agentJobId = null, status
}
export function updateJobHistory(id, values) {
- db.prepare('UPDATE job_history SET status = ?, error = ?, updated_at = ? WHERE id = ?')
- .run(values.status, values.error || null, new Date().toISOString(), id);
+ const current = db.prepare('SELECT * FROM job_history WHERE id = ?').get(id);
+ if (!current) return;
+ const finishedAt = values.finishedAt ?? current.finished_at;
+ const durationMs = finishedAt && current.started_at ? new Date(finishedAt).getTime() - new Date(current.started_at).getTime() : current.duration_ms;
+ db.prepare(`
+ UPDATE job_history
+ SET status = ?, error = ?, current_step = ?, snapshot_id = ?, finished_at = ?, duration_ms = ?, updated_at = ?
+ WHERE id = ?
+ `).run(
+ values.status ?? current.status,
+ values.error ?? current.error,
+ values.currentStep ?? current.current_step,
+ values.snapshotId ?? current.snapshot_id,
+ finishedAt,
+ Number.isFinite(durationMs) ? durationMs : null,
+ new Date().toISOString(),
+ id,
+ );
+}
+
+export function listOpenJobHistory() {
+ return db.prepare(`
+ SELECT job_history.*, nodes.base_url, nodes.token
+ FROM job_history
+ JOIN nodes ON nodes.id = job_history.node_id
+ WHERE job_history.agent_job_id IS NOT NULL
+ AND job_history.status IN ('accepted', 'queued', 'running')
+ AND nodes.enabled = 1
+ ORDER BY job_history.started_at ASC
+ `).all();
}
export function listJobHistory(limit = 100) {
@@ -122,7 +150,11 @@ export function listJobHistory(limit = 100) {
agentJobId: row.agent_job_id,
status: row.status,
error: row.error,
+ currentStep: row.current_step,
+ snapshotId: row.snapshot_id,
startedAt: row.started_at,
+ finishedAt: row.finished_at,
+ durationMs: row.duration_ms,
updatedAt: row.updated_at,
}));
}