Issue 1: finaler Agent-Job-Status
Management pollt jetzt offene Agent-Jobs und aktualisiert job_history mit: - finalem success / failed - Fehlertext - aktuellem/finalem Step - finishedAt - durationMs - Snapshot-ID aus den Agent-Logs, wenn ein Backup erfolgreich war Die Operations-Seite zeigt jetzt Dauer und Snapshot-ID. Issue 2: erweiterte Agent-Healthchecks /api/health am Agent prüft jetzt strukturiert: - fehlende Env-Konfiguration - Commands: incus, zfs, zpool, restic, udevadm, dd - ZFS Pool Existenz - ZFS Pool Capacity/Freespace via zpool list - /dev/zvol Zugriff - Restic Repository Zugriff
This commit is contained in:
@@ -26,6 +26,7 @@ Required commands:
|
|||||||
|
|
||||||
- `incus`
|
- `incus`
|
||||||
- `zfs`
|
- `zfs`
|
||||||
|
- `zpool`
|
||||||
- `restic`
|
- `restic`
|
||||||
- `udevadm`
|
- `udevadm`
|
||||||
- `dd`
|
- `dd`
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
import { Router } from 'express';
|
import { Router } from 'express';
|
||||||
import { missingEnvVars } from '../config.js';
|
import { access } from 'node:fs/promises';
|
||||||
|
import { constants } from 'node:fs';
|
||||||
|
import { config, missingEnvVars } from '../config.js';
|
||||||
import { spawnCommand, runRestic } from '../executor.js';
|
import { spawnCommand, runRestic } from '../executor.js';
|
||||||
|
|
||||||
export const healthRouter = Router();
|
export const healthRouter = Router();
|
||||||
@@ -7,35 +9,73 @@ export const healthRouter = Router();
|
|||||||
healthRouter.get('/', async (_req, res) => {
|
healthRouter.get('/', async (_req, res) => {
|
||||||
const missing = missingEnvVars();
|
const missing = missingEnvVars();
|
||||||
const checks = {
|
const checks = {
|
||||||
config: missing.length ? `missing: ${missing.join(', ')}` : 'ok',
|
config: checkValue(!missing.length, missing.length ? `missing: ${missing.join(', ')}` : 'ok'),
|
||||||
incus: 'pending',
|
commands: await checkCommands(['incus', 'zfs', 'zpool', 'restic', 'udevadm', 'dd']),
|
||||||
zfs: 'pending',
|
zfsPool: await checkZfsPool(),
|
||||||
restic: 'pending',
|
zvol: await checkZvolAccess(),
|
||||||
|
resticRepository: await checkResticRepository(),
|
||||||
};
|
};
|
||||||
|
|
||||||
await Promise.all([
|
const ok = Object.values(checks).every((value) => value.ok);
|
||||||
checkCommand('incus', ['version']).then((value) => {
|
|
||||||
checks.incus = value;
|
|
||||||
}),
|
|
||||||
checkCommand('zfs', ['version']).then((value) => {
|
|
||||||
checks.zfs = value;
|
|
||||||
}),
|
|
||||||
runRestic(['snapshots', '--json'], { ignoreExitCode: true }).then((result) => {
|
|
||||||
checks.restic = result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed';
|
|
||||||
}).catch((error) => {
|
|
||||||
checks.restic = error.message;
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const ok = Object.values(checks).every((value) => value === 'ok');
|
|
||||||
res.status(ok ? 200 : 503).json({ ok, checks });
|
res.status(ok ? 200 : 503).json({ ok, checks });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
async function checkCommands(commands) {
|
||||||
|
const results = {};
|
||||||
|
await Promise.all(commands.map(async (command) => {
|
||||||
|
results[command] = await checkCommand(command, ['--version']);
|
||||||
|
}));
|
||||||
|
const failed = Object.entries(results).filter(([, result]) => !result.ok);
|
||||||
|
return {
|
||||||
|
ok: failed.length === 0,
|
||||||
|
message: failed.length ? `failed: ${failed.map(([command]) => command).join(', ')}` : 'ok',
|
||||||
|
details: results,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function checkCommand(command, args) {
|
async function checkCommand(command, args) {
|
||||||
try {
|
try {
|
||||||
const result = await spawnCommand(command, args, { ignoreExitCode: true });
|
const result = await spawnCommand(command, args, { ignoreExitCode: true });
|
||||||
return result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed';
|
return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || result.stdout.trim() || 'failed');
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return error.message;
|
return checkValue(false, error.message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function checkZfsPool() {
|
||||||
|
if (!config.zfsPoolName) return checkValue(false, 'ZFS_POOL_NAME is not configured');
|
||||||
|
try {
|
||||||
|
const result = await spawnCommand('zpool', ['list', '-Hp', '-o', 'name,size,alloc,free,cap', config.zfsPoolName], { ignoreExitCode: true });
|
||||||
|
if (result.exitCode !== 0) return checkValue(false, result.stderr.trim() || 'zpool list failed');
|
||||||
|
const [name, size, allocated, free, capacity] = result.stdout.trim().split('\t');
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
message: 'ok',
|
||||||
|
details: { name, size: Number(size), allocated: Number(allocated), free: Number(free), capacity },
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
return checkValue(false, error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function checkZvolAccess() {
|
||||||
|
try {
|
||||||
|
await access('/dev/zvol', constants.R_OK);
|
||||||
|
return checkValue(true, 'ok');
|
||||||
|
} catch (error) {
|
||||||
|
return checkValue(false, error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function checkResticRepository() {
|
||||||
|
try {
|
||||||
|
const result = await runRestic(['snapshots', '--json'], { ignoreExitCode: true });
|
||||||
|
return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'restic repository check failed');
|
||||||
|
} catch (error) {
|
||||||
|
return checkValue(false, error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function checkValue(ok, message, details = null) {
|
||||||
|
return { ok, message, ...(details ? { details } : {}) };
|
||||||
|
}
|
||||||
|
|||||||
+1
-1
@@ -32,7 +32,7 @@ Make `/api/health` useful for diagnosing whether a node can actually run backup
|
|||||||
|
|
||||||
### Tasks
|
### Tasks
|
||||||
|
|
||||||
- Check that required commands exist: `incus`, `zfs`, `restic`, `udevadm`, `dd`.
|
- Check that required commands exist: `incus`, `zfs`, `zpool`, `restic`, `udevadm`, `dd`.
|
||||||
- Check that configured ZFS pool exists.
|
- Check that configured ZFS pool exists.
|
||||||
- Check that `/dev/zvol` is accessible.
|
- Check that `/dev/zvol` is accessible.
|
||||||
- Check Restic repository access.
|
- Check Restic repository access.
|
||||||
|
|||||||
@@ -31,13 +31,15 @@ export function Operations() {
|
|||||||
</div>
|
</div>
|
||||||
{error ? <div className="rounded-md border border-red-900 bg-red-950/50 px-4 py-3 text-sm text-red-200">{error}</div> : null}
|
{error ? <div className="rounded-md border border-red-900 bg-red-950/50 px-4 py-3 text-sm text-red-200">{error}</div> : null}
|
||||||
<Table
|
<Table
|
||||||
columns={['Time', 'Type', 'Node', 'VM', 'Status', 'Agent Job']}
|
columns={['Time', 'Type', 'Node', 'VM', 'Status', 'Duration', 'Snapshot', 'Agent Job']}
|
||||||
rows={history.map((row) => [
|
rows={history.map((row) => [
|
||||||
formatTime(row.startedAt),
|
formatTime(row.startedAt),
|
||||||
row.type,
|
row.type,
|
||||||
row.nodeName,
|
row.nodeName,
|
||||||
row.vmName,
|
row.vmName,
|
||||||
row.error || row.status,
|
row.error || row.status,
|
||||||
|
formatDuration(row.durationMs),
|
||||||
|
row.snapshotId || '-',
|
||||||
row.agentJobId || '-',
|
row.agentJobId || '-',
|
||||||
])}
|
])}
|
||||||
title="Job History"
|
title="Job History"
|
||||||
@@ -86,3 +88,10 @@ function formatTime(value) {
|
|||||||
if (!value) return '-';
|
if (!value) return '-';
|
||||||
return new Intl.DateTimeFormat(undefined, { dateStyle: 'short', timeStyle: 'medium' }).format(new Date(value));
|
return new Intl.DateTimeFormat(undefined, { dateStyle: 'short', timeStyle: 'medium' }).format(new Date(value));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatDuration(value) {
|
||||||
|
if (!value) return '-';
|
||||||
|
const seconds = Math.round(Number(value) / 1000);
|
||||||
|
if (seconds < 60) return `${seconds}s`;
|
||||||
|
return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
|
||||||
|
}
|
||||||
|
|||||||
@@ -65,10 +65,18 @@ export function migrate() {
|
|||||||
agent_job_id TEXT,
|
agent_job_id TEXT,
|
||||||
status TEXT NOT NULL,
|
status TEXT NOT NULL,
|
||||||
error TEXT,
|
error TEXT,
|
||||||
|
current_step TEXT,
|
||||||
|
snapshot_id TEXT,
|
||||||
started_at TEXT NOT NULL,
|
started_at TEXT NOT NULL,
|
||||||
|
finished_at TEXT,
|
||||||
|
duration_ms INTEGER,
|
||||||
updated_at TEXT NOT NULL
|
updated_at TEXT NOT NULL
|
||||||
);
|
);
|
||||||
`);
|
`);
|
||||||
|
addColumnIfMissing('job_history', 'current_step', 'TEXT');
|
||||||
|
addColumnIfMissing('job_history', 'snapshot_id', 'TEXT');
|
||||||
|
addColumnIfMissing('job_history', 'finished_at', 'TEXT');
|
||||||
|
addColumnIfMissing('job_history', 'duration_ms', 'INTEGER');
|
||||||
|
|
||||||
const existing = db.prepare('SELECT id FROM users LIMIT 1').get();
|
const existing = db.prepare('SELECT id FROM users LIMIT 1').get();
|
||||||
if (!existing) {
|
if (!existing) {
|
||||||
@@ -82,3 +90,10 @@ export function migrate() {
|
|||||||
function cryptoId() {
|
function cryptoId() {
|
||||||
return Math.random().toString(16).slice(2) + Date.now().toString(16);
|
return Math.random().toString(16).slice(2) + Date.now().toString(16);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function addColumnIfMissing(table, column, definition) {
|
||||||
|
const columns = db.prepare(`PRAGMA table_info(${table})`).all().map((row) => row.name);
|
||||||
|
if (!columns.includes(column)) {
|
||||||
|
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import express from 'express';
|
|||||||
import { config } from './config.js';
|
import { config } from './config.js';
|
||||||
import { migrate } from './db.js';
|
import { migrate } from './db.js';
|
||||||
import { requireAuth } from './auth.js';
|
import { requireAuth } from './auth.js';
|
||||||
|
import { startJobPoller } from './jobPoller.js';
|
||||||
import { authRouter } from './routes/auth.js';
|
import { authRouter } from './routes/auth.js';
|
||||||
import { nodesRouter } from './routes/nodes.js';
|
import { nodesRouter } from './routes/nodes.js';
|
||||||
import { operationsRouter } from './routes/operations.js';
|
import { operationsRouter } from './routes/operations.js';
|
||||||
@@ -33,3 +34,4 @@ app.listen(config.port, () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
startScheduler();
|
startScheduler();
|
||||||
|
startJobPoller();
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
import { agentRequest } from './agentClient.js';
|
||||||
|
import { listOpenJobHistory, updateJobHistory } from './store.js';
|
||||||
|
|
||||||
|
let timer = null;
|
||||||
|
|
||||||
|
export function startJobPoller() {
|
||||||
|
if (timer) return;
|
||||||
|
timer = setInterval(pollJobs, 5000);
|
||||||
|
timer.unref?.();
|
||||||
|
pollJobs();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pollJobs() {
|
||||||
|
const jobs = listOpenJobHistory();
|
||||||
|
await Promise.all(jobs.map(async (history) => {
|
||||||
|
try {
|
||||||
|
const job = await agentRequest({
|
||||||
|
id: history.node_id,
|
||||||
|
baseUrl: history.base_url,
|
||||||
|
token: history.token,
|
||||||
|
}, `/jobs/${encodeURIComponent(history.agent_job_id)}`, { timeout: 10000 });
|
||||||
|
|
||||||
|
updateJobHistory(history.id, {
|
||||||
|
status: job.status,
|
||||||
|
error: job.error,
|
||||||
|
currentStep: job.currentStep,
|
||||||
|
snapshotId: findSnapshotId(job),
|
||||||
|
finishedAt: job.finishedAt,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
updateJobHistory(history.id, {
|
||||||
|
status: 'failed',
|
||||||
|
error: error.message,
|
||||||
|
finishedAt: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function findSnapshotId(job) {
|
||||||
|
const line = [...(job.logs || [])].reverse().find((entry) => /^snapshot [a-f0-9]+ saved$/i.test(entry));
|
||||||
|
return line ? line.split(' ')[1] : null;
|
||||||
|
}
|
||||||
+34
-2
@@ -108,8 +108,36 @@ export function createJobHistory({ node, vmName, type, agentJobId = null, status
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function updateJobHistory(id, values) {
|
export function updateJobHistory(id, values) {
|
||||||
db.prepare('UPDATE job_history SET status = ?, error = ?, updated_at = ? WHERE id = ?')
|
const current = db.prepare('SELECT * FROM job_history WHERE id = ?').get(id);
|
||||||
.run(values.status, values.error || null, new Date().toISOString(), id);
|
if (!current) return;
|
||||||
|
const finishedAt = values.finishedAt ?? current.finished_at;
|
||||||
|
const durationMs = finishedAt && current.started_at ? new Date(finishedAt).getTime() - new Date(current.started_at).getTime() : current.duration_ms;
|
||||||
|
db.prepare(`
|
||||||
|
UPDATE job_history
|
||||||
|
SET status = ?, error = ?, current_step = ?, snapshot_id = ?, finished_at = ?, duration_ms = ?, updated_at = ?
|
||||||
|
WHERE id = ?
|
||||||
|
`).run(
|
||||||
|
values.status ?? current.status,
|
||||||
|
values.error ?? current.error,
|
||||||
|
values.currentStep ?? current.current_step,
|
||||||
|
values.snapshotId ?? current.snapshot_id,
|
||||||
|
finishedAt,
|
||||||
|
Number.isFinite(durationMs) ? durationMs : null,
|
||||||
|
new Date().toISOString(),
|
||||||
|
id,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function listOpenJobHistory() {
|
||||||
|
return db.prepare(`
|
||||||
|
SELECT job_history.*, nodes.base_url, nodes.token
|
||||||
|
FROM job_history
|
||||||
|
JOIN nodes ON nodes.id = job_history.node_id
|
||||||
|
WHERE job_history.agent_job_id IS NOT NULL
|
||||||
|
AND job_history.status IN ('accepted', 'queued', 'running')
|
||||||
|
AND nodes.enabled = 1
|
||||||
|
ORDER BY job_history.started_at ASC
|
||||||
|
`).all();
|
||||||
}
|
}
|
||||||
|
|
||||||
export function listJobHistory(limit = 100) {
|
export function listJobHistory(limit = 100) {
|
||||||
@@ -122,7 +150,11 @@ export function listJobHistory(limit = 100) {
|
|||||||
agentJobId: row.agent_job_id,
|
agentJobId: row.agent_job_id,
|
||||||
status: row.status,
|
status: row.status,
|
||||||
error: row.error,
|
error: row.error,
|
||||||
|
currentStep: row.current_step,
|
||||||
|
snapshotId: row.snapshot_id,
|
||||||
startedAt: row.started_at,
|
startedAt: row.started_at,
|
||||||
|
finishedAt: row.finished_at,
|
||||||
|
durationMs: row.duration_ms,
|
||||||
updatedAt: row.updated_at,
|
updatedAt: row.updated_at,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user