Issue 1: finaler Agent-Job-Status

Management pollt jetzt offene Agent-Jobs und aktualisiert job_history mit:

  - finalem success / failed
  - Fehlertext
  - aktuellem/finalem Step
  - finishedAt
  - durationMs
  - Snapshot-ID aus den Agent-Logs, wenn ein Backup erfolgreich war

  Die Operations-Seite zeigt jetzt Dauer und Snapshot-ID.

  Issue 2: erweiterte Agent-Healthchecks
  /api/health am Agent prüft jetzt strukturiert:

  - fehlende Env-Konfiguration
  - Commands: incus, zfs, zpool, restic, udevadm, dd
  - ZFS Pool Existenz
  - ZFS Pool Capacity/Freespace via zpool list
  - /dev/zvol Zugriff
  - Restic Repository Zugriff
This commit is contained in:
Philipp
2026-05-21 10:27:04 +02:00
parent dc4989acfb
commit 87682a777f
8 changed files with 168 additions and 26 deletions
+1
View File
@@ -26,6 +26,7 @@ Required commands:
- `incus` - `incus`
- `zfs` - `zfs`
- `zpool`
- `restic` - `restic`
- `udevadm` - `udevadm`
- `dd` - `dd`
+62 -22
View File
@@ -1,5 +1,7 @@
import { Router } from 'express'; import { Router } from 'express';
import { missingEnvVars } from '../config.js'; import { access } from 'node:fs/promises';
import { constants } from 'node:fs';
import { config, missingEnvVars } from '../config.js';
import { spawnCommand, runRestic } from '../executor.js'; import { spawnCommand, runRestic } from '../executor.js';
export const healthRouter = Router(); export const healthRouter = Router();
@@ -7,35 +9,73 @@ export const healthRouter = Router();
healthRouter.get('/', async (_req, res) => { healthRouter.get('/', async (_req, res) => {
const missing = missingEnvVars(); const missing = missingEnvVars();
const checks = { const checks = {
config: missing.length ? `missing: ${missing.join(', ')}` : 'ok', config: checkValue(!missing.length, missing.length ? `missing: ${missing.join(', ')}` : 'ok'),
incus: 'pending', commands: await checkCommands(['incus', 'zfs', 'zpool', 'restic', 'udevadm', 'dd']),
zfs: 'pending', zfsPool: await checkZfsPool(),
restic: 'pending', zvol: await checkZvolAccess(),
resticRepository: await checkResticRepository(),
}; };
await Promise.all([ const ok = Object.values(checks).every((value) => value.ok);
checkCommand('incus', ['version']).then((value) => {
checks.incus = value;
}),
checkCommand('zfs', ['version']).then((value) => {
checks.zfs = value;
}),
runRestic(['snapshots', '--json'], { ignoreExitCode: true }).then((result) => {
checks.restic = result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed';
}).catch((error) => {
checks.restic = error.message;
}),
]);
const ok = Object.values(checks).every((value) => value === 'ok');
res.status(ok ? 200 : 503).json({ ok, checks }); res.status(ok ? 200 : 503).json({ ok, checks });
}); });
async function checkCommands(commands) {
const results = {};
await Promise.all(commands.map(async (command) => {
results[command] = await checkCommand(command, ['--version']);
}));
const failed = Object.entries(results).filter(([, result]) => !result.ok);
return {
ok: failed.length === 0,
message: failed.length ? `failed: ${failed.map(([command]) => command).join(', ')}` : 'ok',
details: results,
};
}
async function checkCommand(command, args) { async function checkCommand(command, args) {
try { try {
const result = await spawnCommand(command, args, { ignoreExitCode: true }); const result = await spawnCommand(command, args, { ignoreExitCode: true });
return result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed'; return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || result.stdout.trim() || 'failed');
} catch (error) { } catch (error) {
return error.message; return checkValue(false, error.message);
} }
} }
async function checkZfsPool() {
if (!config.zfsPoolName) return checkValue(false, 'ZFS_POOL_NAME is not configured');
try {
const result = await spawnCommand('zpool', ['list', '-Hp', '-o', 'name,size,alloc,free,cap', config.zfsPoolName], { ignoreExitCode: true });
if (result.exitCode !== 0) return checkValue(false, result.stderr.trim() || 'zpool list failed');
const [name, size, allocated, free, capacity] = result.stdout.trim().split('\t');
return {
ok: true,
message: 'ok',
details: { name, size: Number(size), allocated: Number(allocated), free: Number(free), capacity },
};
} catch (error) {
return checkValue(false, error.message);
}
}
async function checkZvolAccess() {
try {
await access('/dev/zvol', constants.R_OK);
return checkValue(true, 'ok');
} catch (error) {
return checkValue(false, error.message);
}
}
async function checkResticRepository() {
try {
const result = await runRestic(['snapshots', '--json'], { ignoreExitCode: true });
return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'restic repository check failed');
} catch (error) {
return checkValue(false, error.message);
}
}
function checkValue(ok, message, details = null) {
return { ok, message, ...(details ? { details } : {}) };
}
+1 -1
View File
@@ -32,7 +32,7 @@ Make `/api/health` useful for diagnosing whether a node can actually run backup
### Tasks ### Tasks
- Check that required commands exist: `incus`, `zfs`, `restic`, `udevadm`, `dd`. - Check that required commands exist: `incus`, `zfs`, `zpool`, `restic`, `udevadm`, `dd`.
- Check that configured ZFS pool exists. - Check that configured ZFS pool exists.
- Check that `/dev/zvol` is accessible. - Check that `/dev/zvol` is accessible.
- Check Restic repository access. - Check Restic repository access.
+10 -1
View File
@@ -31,13 +31,15 @@ export function Operations() {
</div> </div>
{error ? <div className="rounded-md border border-red-900 bg-red-950/50 px-4 py-3 text-sm text-red-200">{error}</div> : null} {error ? <div className="rounded-md border border-red-900 bg-red-950/50 px-4 py-3 text-sm text-red-200">{error}</div> : null}
<Table <Table
columns={['Time', 'Type', 'Node', 'VM', 'Status', 'Agent Job']} columns={['Time', 'Type', 'Node', 'VM', 'Status', 'Duration', 'Snapshot', 'Agent Job']}
rows={history.map((row) => [ rows={history.map((row) => [
formatTime(row.startedAt), formatTime(row.startedAt),
row.type, row.type,
row.nodeName, row.nodeName,
row.vmName, row.vmName,
row.error || row.status, row.error || row.status,
formatDuration(row.durationMs),
row.snapshotId || '-',
row.agentJobId || '-', row.agentJobId || '-',
])} ])}
title="Job History" title="Job History"
@@ -86,3 +88,10 @@ function formatTime(value) {
if (!value) return '-'; if (!value) return '-';
return new Intl.DateTimeFormat(undefined, { dateStyle: 'short', timeStyle: 'medium' }).format(new Date(value)); return new Intl.DateTimeFormat(undefined, { dateStyle: 'short', timeStyle: 'medium' }).format(new Date(value));
} }
function formatDuration(value) {
if (!value) return '-';
const seconds = Math.round(Number(value) / 1000);
if (seconds < 60) return `${seconds}s`;
return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
}
+15
View File
@@ -65,10 +65,18 @@ export function migrate() {
agent_job_id TEXT, agent_job_id TEXT,
status TEXT NOT NULL, status TEXT NOT NULL,
error TEXT, error TEXT,
current_step TEXT,
snapshot_id TEXT,
started_at TEXT NOT NULL, started_at TEXT NOT NULL,
finished_at TEXT,
duration_ms INTEGER,
updated_at TEXT NOT NULL updated_at TEXT NOT NULL
); );
`); `);
addColumnIfMissing('job_history', 'current_step', 'TEXT');
addColumnIfMissing('job_history', 'snapshot_id', 'TEXT');
addColumnIfMissing('job_history', 'finished_at', 'TEXT');
addColumnIfMissing('job_history', 'duration_ms', 'INTEGER');
const existing = db.prepare('SELECT id FROM users LIMIT 1').get(); const existing = db.prepare('SELECT id FROM users LIMIT 1').get();
if (!existing) { if (!existing) {
@@ -82,3 +90,10 @@ export function migrate() {
function cryptoId() { function cryptoId() {
return Math.random().toString(16).slice(2) + Date.now().toString(16); return Math.random().toString(16).slice(2) + Date.now().toString(16);
} }
function addColumnIfMissing(table, column, definition) {
const columns = db.prepare(`PRAGMA table_info(${table})`).all().map((row) => row.name);
if (!columns.includes(column)) {
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
}
}
+2
View File
@@ -3,6 +3,7 @@ import express from 'express';
import { config } from './config.js'; import { config } from './config.js';
import { migrate } from './db.js'; import { migrate } from './db.js';
import { requireAuth } from './auth.js'; import { requireAuth } from './auth.js';
import { startJobPoller } from './jobPoller.js';
import { authRouter } from './routes/auth.js'; import { authRouter } from './routes/auth.js';
import { nodesRouter } from './routes/nodes.js'; import { nodesRouter } from './routes/nodes.js';
import { operationsRouter } from './routes/operations.js'; import { operationsRouter } from './routes/operations.js';
@@ -33,3 +34,4 @@ app.listen(config.port, () => {
}); });
startScheduler(); startScheduler();
startJobPoller();
+43
View File
@@ -0,0 +1,43 @@
import { agentRequest } from './agentClient.js';
import { listOpenJobHistory, updateJobHistory } from './store.js';
let timer = null;
export function startJobPoller() {
if (timer) return;
timer = setInterval(pollJobs, 5000);
timer.unref?.();
pollJobs();
}
async function pollJobs() {
const jobs = listOpenJobHistory();
await Promise.all(jobs.map(async (history) => {
try {
const job = await agentRequest({
id: history.node_id,
baseUrl: history.base_url,
token: history.token,
}, `/jobs/${encodeURIComponent(history.agent_job_id)}`, { timeout: 10000 });
updateJobHistory(history.id, {
status: job.status,
error: job.error,
currentStep: job.currentStep,
snapshotId: findSnapshotId(job),
finishedAt: job.finishedAt,
});
} catch (error) {
updateJobHistory(history.id, {
status: 'failed',
error: error.message,
finishedAt: new Date().toISOString(),
});
}
}));
}
function findSnapshotId(job) {
const line = [...(job.logs || [])].reverse().find((entry) => /^snapshot [a-f0-9]+ saved$/i.test(entry));
return line ? line.split(' ')[1] : null;
}
+34 -2
View File
@@ -108,8 +108,36 @@ export function createJobHistory({ node, vmName, type, agentJobId = null, status
} }
export function updateJobHistory(id, values) { export function updateJobHistory(id, values) {
db.prepare('UPDATE job_history SET status = ?, error = ?, updated_at = ? WHERE id = ?') const current = db.prepare('SELECT * FROM job_history WHERE id = ?').get(id);
.run(values.status, values.error || null, new Date().toISOString(), id); if (!current) return;
const finishedAt = values.finishedAt ?? current.finished_at;
const durationMs = finishedAt && current.started_at ? new Date(finishedAt).getTime() - new Date(current.started_at).getTime() : current.duration_ms;
db.prepare(`
UPDATE job_history
SET status = ?, error = ?, current_step = ?, snapshot_id = ?, finished_at = ?, duration_ms = ?, updated_at = ?
WHERE id = ?
`).run(
values.status ?? current.status,
values.error ?? current.error,
values.currentStep ?? current.current_step,
values.snapshotId ?? current.snapshot_id,
finishedAt,
Number.isFinite(durationMs) ? durationMs : null,
new Date().toISOString(),
id,
);
}
export function listOpenJobHistory() {
return db.prepare(`
SELECT job_history.*, nodes.base_url, nodes.token
FROM job_history
JOIN nodes ON nodes.id = job_history.node_id
WHERE job_history.agent_job_id IS NOT NULL
AND job_history.status IN ('accepted', 'queued', 'running')
AND nodes.enabled = 1
ORDER BY job_history.started_at ASC
`).all();
} }
export function listJobHistory(limit = 100) { export function listJobHistory(limit = 100) {
@@ -122,7 +150,11 @@ export function listJobHistory(limit = 100) {
agentJobId: row.agent_job_id, agentJobId: row.agent_job_id,
status: row.status, status: row.status,
error: row.error, error: row.error,
currentStep: row.current_step,
snapshotId: row.snapshot_id,
startedAt: row.started_at, startedAt: row.started_at,
finishedAt: row.finished_at,
durationMs: row.duration_ms,
updatedAt: row.updated_at, updatedAt: row.updated_at,
})); }));
} }