Issue 1: finaler Agent-Job-Status

Management pollt jetzt offene Agent-Jobs und aktualisiert job_history mit: - finalem success / failed - Fehlertext - aktuellem/finalem Step - finishedAt - durationMs - Snapshot-ID aus den Agent-Logs, wenn ein Backup erfolgreich war Die Operations-Seite zeigt jetzt Dauer und Snapshot-ID. Issue 2: erweiterte Agent-Healthchecks /api/health am Agent prüft jetzt strukturiert: - fehlende Env-Konfiguration - Commands: incus, zfs, zpool, restic, udevadm, dd - ZFS Pool Existenz - ZFS Pool Capacity/Freespace via zpool list - /dev/zvol Zugriff - Restic Repository Zugriff
2026-05-21 10:27:04 +02:00
parent dc4989acfb
commit 87682a777f
8 changed files with 168 additions and 26 deletions
@@ -26,6 +26,7 @@ Required commands:

 - `incus`
 - `zfs`
+- `zpool`
 - `restic`
 - `udevadm`
 - `dd`
@@ -1,5 +1,7 @@
 import { Router } from 'express';
-import { missingEnvVars } from '../config.js';
+import { access } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { config, missingEnvVars } from '../config.js';
 import { spawnCommand, runRestic } from '../executor.js';

 export const healthRouter = Router();
@@ -7,35 +9,73 @@ export const healthRouter = Router();
 healthRouter.get('/', async (_req, res) => {
  const missing = missingEnvVars();
  const checks = {
-    config: missing.length ? `missing: ${missing.join(', ')}` : 'ok',
-    incus: 'pending',
-    zfs: 'pending',
-    restic: 'pending',
+    config: checkValue(!missing.length, missing.length ? `missing: ${missing.join(', ')}` : 'ok'),
+    commands: await checkCommands(['incus', 'zfs', 'zpool', 'restic', 'udevadm', 'dd']),
+    zfsPool: await checkZfsPool(),
+    zvol: await checkZvolAccess(),
+    resticRepository: await checkResticRepository(),
  };

-  await Promise.all([
-    checkCommand('incus', ['version']).then((value) => {
-      checks.incus = value;
-    }),
-    checkCommand('zfs', ['version']).then((value) => {
-      checks.zfs = value;
-    }),
-    runRestic(['snapshots', '--json'], { ignoreExitCode: true }).then((result) => {
-      checks.restic = result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed';
-    }).catch((error) => {
-      checks.restic = error.message;
-    }),
-  ]);
-
-  const ok = Object.values(checks).every((value) => value === 'ok');
+  const ok = Object.values(checks).every((value) => value.ok);
  res.status(ok ? 200 : 503).json({ ok, checks });
 });

+async function checkCommands(commands) {
+  const results = {};
+  await Promise.all(commands.map(async (command) => {
+    results[command] = await checkCommand(command, ['--version']);
+  }));
+  const failed = Object.entries(results).filter(([, result]) => !result.ok);
+  return {
+    ok: failed.length === 0,
+    message: failed.length ? `failed: ${failed.map(([command]) => command).join(', ')}` : 'ok',
+    details: results,
+  };
+}
+
 async function checkCommand(command, args) {
  try {
    const result = await spawnCommand(command, args, { ignoreExitCode: true });
-    return result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed';
+    return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || result.stdout.trim() || 'failed');
  } catch (error) {
-    return error.message;
+    return checkValue(false, error.message);
  }
 }
+
+async function checkZfsPool() {
+  if (!config.zfsPoolName) return checkValue(false, 'ZFS_POOL_NAME is not configured');
+  try {
+    const result = await spawnCommand('zpool', ['list', '-Hp', '-o', 'name,size,alloc,free,cap', config.zfsPoolName], { ignoreExitCode: true });
+    if (result.exitCode !== 0) return checkValue(false, result.stderr.trim() || 'zpool list failed');
+    const [name, size, allocated, free, capacity] = result.stdout.trim().split('\t');
+    return {
+      ok: true,
+      message: 'ok',
+      details: { name, size: Number(size), allocated: Number(allocated), free: Number(free), capacity },
+    };
+  } catch (error) {
+    return checkValue(false, error.message);
+  }
+}
+
+async function checkZvolAccess() {
+  try {
+    await access('/dev/zvol', constants.R_OK);
+    return checkValue(true, 'ok');
+  } catch (error) {
+    return checkValue(false, error.message);
+  }
+}
+
+async function checkResticRepository() {
+  try {
+    const result = await runRestic(['snapshots', '--json'], { ignoreExitCode: true });
+    return checkValue(result.exitCode === 0, result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'restic repository check failed');
+  } catch (error) {
+    return checkValue(false, error.message);
+  }
+}
+
+function checkValue(ok, message, details = null) {
+  return { ok, message, ...(details ? { details } : {}) };
+}
@@ -32,7 +32,7 @@ Make `/api/health` useful for diagnosing whether a node can actually run backup

 ### Tasks

- Check that required commands exist: `incus`, `zfs`, `restic`, `udevadm`, `dd`.
+- Check that required commands exist: `incus`, `zfs`, `zpool`, `restic`, `udevadm`, `dd`.
 - Check that configured ZFS pool exists.
 - Check that `/dev/zvol` is accessible.
 - Check Restic repository access.
@@ -31,13 +31,15 @@ export function Operations() {
      </div>
      {error ? <div className="rounded-md border border-red-900 bg-red-950/50 px-4 py-3 text-sm text-red-200">{error}</div> : null}
      <Table
-        columns={['Time', 'Type', 'Node', 'VM', 'Status', 'Agent Job']}
+        columns={['Time', 'Type', 'Node', 'VM', 'Status', 'Duration', 'Snapshot', 'Agent Job']}
        rows={history.map((row) => [
          formatTime(row.startedAt),
          row.type,
          row.nodeName,
          row.vmName,
          row.error || row.status,
+          formatDuration(row.durationMs),
+          row.snapshotId || '-',
          row.agentJobId || '-',
        ])}
        title="Job History"
@@ -86,3 +88,10 @@ function formatTime(value) {
  if (!value) return '-';
  return new Intl.DateTimeFormat(undefined, { dateStyle: 'short', timeStyle: 'medium' }).format(new Date(value));
 }
+
+function formatDuration(value) {
+  if (!value) return '-';
+  const seconds = Math.round(Number(value) / 1000);
+  if (seconds < 60) return `${seconds}s`;
+  return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
+}
@@ -65,10 +65,18 @@ export function migrate() {
      agent_job_id TEXT,
      status TEXT NOT NULL,
      error TEXT,
+      current_step TEXT,
+      snapshot_id TEXT,
      started_at TEXT NOT NULL,
+      finished_at TEXT,
+      duration_ms INTEGER,
      updated_at TEXT NOT NULL
    );
  `);
+  addColumnIfMissing('job_history', 'current_step', 'TEXT');
+  addColumnIfMissing('job_history', 'snapshot_id', 'TEXT');
+  addColumnIfMissing('job_history', 'finished_at', 'TEXT');
+  addColumnIfMissing('job_history', 'duration_ms', 'INTEGER');

  const existing = db.prepare('SELECT id FROM users LIMIT 1').get();
  if (!existing) {
@@ -82,3 +90,10 @@ export function migrate() {
 function cryptoId() {
  return Math.random().toString(16).slice(2) + Date.now().toString(16);
 }
+
+function addColumnIfMissing(table, column, definition) {
+  const columns = db.prepare(`PRAGMA table_info(${table})`).all().map((row) => row.name);
+  if (!columns.includes(column)) {
+    db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
+  }
+}
@@ -3,6 +3,7 @@ import express from 'express';
 import { config } from './config.js';
 import { migrate } from './db.js';
 import { requireAuth } from './auth.js';
+import { startJobPoller } from './jobPoller.js';
 import { authRouter } from './routes/auth.js';
 import { nodesRouter } from './routes/nodes.js';
 import { operationsRouter } from './routes/operations.js';
@@ -33,3 +34,4 @@ app.listen(config.port, () => {
 });

 startScheduler();
+startJobPoller();
@@ -0,0 +1,43 @@
+import { agentRequest } from './agentClient.js';
+import { listOpenJobHistory, updateJobHistory } from './store.js';
+
+let timer = null;
+
+export function startJobPoller() {
+  if (timer) return;
+  timer = setInterval(pollJobs, 5000);
+  timer.unref?.();
+  pollJobs();
+}
+
+async function pollJobs() {
+  const jobs = listOpenJobHistory();
+  await Promise.all(jobs.map(async (history) => {
+    try {
+      const job = await agentRequest({
+        id: history.node_id,
+        baseUrl: history.base_url,
+        token: history.token,
+      }, `/jobs/${encodeURIComponent(history.agent_job_id)}`, { timeout: 10000 });
+
+      updateJobHistory(history.id, {
+        status: job.status,
+        error: job.error,
+        currentStep: job.currentStep,
+        snapshotId: findSnapshotId(job),
+        finishedAt: job.finishedAt,
+      });
+    } catch (error) {
+      updateJobHistory(history.id, {
+        status: 'failed',
+        error: error.message,
+        finishedAt: new Date().toISOString(),
+      });
+    }
+  }));
+}
+
+function findSnapshotId(job) {
+  const line = [...(job.logs || [])].reverse().find((entry) => /^snapshot [a-f0-9]+ saved$/i.test(entry));
+  return line ? line.split(' ')[1] : null;
+}
@@ -108,8 +108,36 @@ export function createJobHistory({ node, vmName, type, agentJobId = null, status
 }

 export function updateJobHistory(id, values) {
-  db.prepare('UPDATE job_history SET status = ?, error = ?, updated_at = ? WHERE id = ?')
-    .run(values.status, values.error || null, new Date().toISOString(), id);
+  const current = db.prepare('SELECT * FROM job_history WHERE id = ?').get(id);
+  if (!current) return;
+  const finishedAt = values.finishedAt ?? current.finished_at;
+  const durationMs = finishedAt && current.started_at ? new Date(finishedAt).getTime() - new Date(current.started_at).getTime() : current.duration_ms;
+  db.prepare(`
+    UPDATE job_history
+    SET status = ?, error = ?, current_step = ?, snapshot_id = ?, finished_at = ?, duration_ms = ?, updated_at = ?
+    WHERE id = ?
+  `).run(
+    values.status ?? current.status,
+    values.error ?? current.error,
+    values.currentStep ?? current.current_step,
+    values.snapshotId ?? current.snapshot_id,
+    finishedAt,
+    Number.isFinite(durationMs) ? durationMs : null,
+    new Date().toISOString(),
+    id,
+  );
+}
+
+export function listOpenJobHistory() {
+  return db.prepare(`
+    SELECT job_history.*, nodes.base_url, nodes.token
+    FROM job_history
+    JOIN nodes ON nodes.id = job_history.node_id
+    WHERE job_history.agent_job_id IS NOT NULL
+      AND job_history.status IN ('accepted', 'queued', 'running')
+      AND nodes.enabled = 1
+    ORDER BY job_history.started_at ASC
+  `).all();
 }

 export function listJobHistory(limit = 100) {
@@ -122,7 +150,11 @@ export function listJobHistory(limit = 100) {
    agentJobId: row.agent_job_id,
    status: row.status,
    error: row.error,
+    currentStep: row.current_step,
+    snapshotId: row.snapshot_id,
    startedAt: row.started_at,
+    finishedAt: row.finished_at,
+    durationMs: row.duration_ms,
    updatedAt: row.updated_at,
  }));
 }