diff --git a/README.md b/README.md index 73a40c8..f091b29 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,15 @@ Set `API_TOKEN` in `agent/.env`; the management server uses that token when call The agent persists recent jobs in SQLite through `AGENT_DATABASE_PATH` (default: `./agent.sqlite`). If the agent restarts while a job is `queued` or `running`, that job is marked `failed` on startup so management can poll a final state and stale VM locks are not kept. +The installer clones only the agent-related repository paths and installs `incubator` for updates and diagnostics: + +```bash +sudo incubator update +sudo incubator status +sudo incubator logs +sudo incubator doctor +``` + The agent can serve HTTPS directly for private networks: ```env diff --git a/agent/.env.example b/agent/.env.example index a55fc1b..2df8047 100644 --- a/agent/.env.example +++ b/agent/.env.example @@ -7,6 +7,7 @@ RESTIC_KEEP_HOURLY=0 RESTIC_KEEP_DAILY=7 RESTIC_KEEP_WEEKLY=0 RESTIC_KEEP_MONTHLY=0 +RESTIC_RETRY_LOCK="5m" HOST="0.0.0.0" PORT=3000 API_TOKEN="change-me-to-at-least-32-characters" diff --git a/agent/src/config.js b/agent/src/config.js index 5ea39b4..dbc461e 100644 --- a/agent/src/config.js +++ b/agent/src/config.js @@ -34,6 +34,7 @@ export const config = { RESTIC_REPOSITORY: process.env.RESTIC_REPOSITORY || '', RESTIC_PASSWORD: process.env.RESTIC_PASSWORD || '', }, + resticRetryLock: process.env.RESTIC_RETRY_LOCK || '5m', retention: { keepHourly: Number(process.env.RESTIC_KEEP_HOURLY || 0), keepDaily: Number(process.env.RESTIC_KEEP_DAILY || 7), @@ -60,6 +61,7 @@ export const editableEnv = [ { key: 'RESTIC_KEEP_DAILY', label: 'Keep daily snapshots', required: false, secret: false }, { key: 'RESTIC_KEEP_WEEKLY', label: 'Keep weekly snapshots', required: false, secret: false }, { key: 'RESTIC_KEEP_MONTHLY', label: 'Keep monthly snapshots', required: false, secret: false }, + { key: 'RESTIC_RETRY_LOCK', label: 'Restic lock retry duration', required: false, secret: false }, { key: 'PORT', label: 'API port', required: false, secret: false }, { key: 'HOST', label: 'API bind host', required: false, secret: false }, { key: 'API_TOKEN', label: 'API token', required: true, secret: true }, @@ -161,6 +163,7 @@ function applyRuntimeEnv(values) { config.resticEnv.AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY || ''; config.resticEnv.RESTIC_REPOSITORY = process.env.RESTIC_REPOSITORY || ''; config.resticEnv.RESTIC_PASSWORD = process.env.RESTIC_PASSWORD || ''; + config.resticRetryLock = process.env.RESTIC_RETRY_LOCK || '5m'; config.retention.keepHourly = Number(process.env.RESTIC_KEEP_HOURLY || 0); config.retention.keepDaily = Number(process.env.RESTIC_KEEP_DAILY || 7); config.retention.keepWeekly = Number(process.env.RESTIC_KEEP_WEEKLY || 0); diff --git a/agent/src/executor.js b/agent/src/executor.js index 2f20389..79a20b9 100644 --- a/agent/src/executor.js +++ b/agent/src/executor.js @@ -1,6 +1,6 @@ import { spawn } from 'node:child_process'; import { PassThrough, Transform } from 'node:stream'; -import { resticProcessEnv } from './config.js'; +import { config, resticProcessEnv } from './config.js'; export class CommandError extends Error { constructor(message, result) { @@ -68,14 +68,14 @@ export function spawnCommand(command, args = [], options = {}) { } export function runRestic(args, options = {}) { - return spawnCommand('restic', args, { + return spawnCommand('restic', resticArgs(args, options), { ...options, env: resticProcessEnv(), }); } export async function resticSnapshotFileSize(snapshotId, filename) { - const result = await runRestic(['ls', '--json', snapshotId]); + const result = await runRestic(['--no-lock', 'ls', '--json', snapshotId], { retryLock: false }); const wanted = `/${filename}`; for (const line of result.stdout.split('\n')) { if (!line.trim()) continue; @@ -89,6 +89,12 @@ export async function resticSnapshotFileSize(snapshotId, filename) { throw error; } +export function resticArgs(args, options = {}) { + if (options.retryLock === false || !config.resticRetryLock) return args; + if (args.includes('--no-lock') || args.includes('--retry-lock')) return args; + return ['--retry-lock', config.resticRetryLock, ...args]; +} + export function createProgressStream(totalBytes, onProgress) { let currentBytes = 0; return new Transform({ diff --git a/agent/src/routes/backup.js b/agent/src/routes/backup.js index d8b51fc..1eab046 100644 --- a/agent/src/routes/backup.js +++ b/agent/src/routes/backup.js @@ -6,7 +6,7 @@ import path from 'node:path'; import { finished } from 'node:stream/promises'; import { setTimeout as delay } from 'node:timers/promises'; import { config } from '../config.js'; -import { createProgressStream, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js'; +import { createProgressStream, resticArgs, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js'; import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js'; import { validateVmExists } from '../validators.js'; @@ -70,7 +70,7 @@ export async function runBackupJob(job, instance = null) { }); }); try { - const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine'], { + const result = await spawnCommand('restic', resticArgs(['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine']), { env: { ...process.env, ...config.resticEnv }, input: snapshotStream.pipe(progressStream), log: (line) => appendJobLog(job, line), @@ -140,7 +140,7 @@ async function runContainerBackupJob(job) { }); let resticOk = false; try { - const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container'], { + const result = await spawnCommand('restic', resticArgs(['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container']), { env: { ...process.env, ...config.resticEnv }, input: zfs.stdout.pipe(progressStream), log: (line) => appendJobLog(job, line), diff --git a/deploy/install-agent.sh b/deploy/install-agent.sh index fdf2275..76bd49f 100755 --- a/deploy/install-agent.sh +++ b/deploy/install-agent.sh @@ -4,11 +4,14 @@ set -eu SERVICE_NAME="incus-backup-agent" DEFAULT_REPO_URL="https://forgejo.digital-droplets.de/philschlo/incus-backup-ui.git" -INSTALL_DIR="${INSTALL_DIR:-/opt/incus-backup-ui}" +INSTALL_DIR="${INSTALL_DIR:-/opt/incubator}" +LEGACY_INSTALL_DIR="${LEGACY_INSTALL_DIR:-/opt/incus-backup-ui}" REPO_URL="${INCUS_BACKUP_REPO_URL:-$DEFAULT_REPO_URL}" BRANCH="${INCUS_BACKUP_BRANCH:-main}" AGENT_DIR="$INSTALL_DIR/agent" OLD_AGENT_DIR="$INSTALL_DIR/backend" +LEGACY_AGENT_DIR="$LEGACY_INSTALL_DIR/agent" +LEGACY_BACKEND_DIR="$LEGACY_INSTALL_DIR/backend" STATE_DIR="${STATE_DIR:-/var/lib/incus-backup-agent}" CONFIG_DIR="${CONFIG_DIR:-/etc/incus-backup-agent}" ENV_FILE="$AGENT_DIR/.env" @@ -58,21 +61,25 @@ copy_repo_from_script_location() { fi command -v rsync >/dev/null 2>&1 || return 1 rsync -a --delete \ - --exclude '.git' \ --exclude 'agent/.env' \ --exclude 'agent/node_modules' \ - --exclude 'management/node_modules' \ - --exclude 'frontend/node_modules' \ - "$repo_dir/" "$INSTALL_DIR/" + "$repo_dir/agent" "$repo_dir/deploy" "$INSTALL_DIR/" +} + +ensure_sparse_checkout() { + git -C "$INSTALL_DIR" sparse-checkout init --cone + git -C "$INSTALL_DIR" sparse-checkout set agent deploy } checkout_or_update_repo() { need_cmd git if [ -d "$INSTALL_DIR/.git" ]; then log "Updating $INSTALL_DIR from Git..." + ensure_sparse_checkout git -C "$INSTALL_DIR" fetch --prune origin "$BRANCH" git -C "$INSTALL_DIR" checkout "$BRANCH" git -C "$INSTALL_DIR" pull --ff-only origin "$BRANCH" + ensure_sparse_checkout return fi @@ -83,7 +90,8 @@ checkout_or_update_repo() { log "Cloning $REPO_URL to $INSTALL_DIR..." mkdir -p "$(dirname "$INSTALL_DIR")" - git clone --branch "$BRANCH" "$REPO_URL" "$INSTALL_DIR" + git clone --filter=blob:none --sparse --branch "$BRANCH" "$REPO_URL" "$INSTALL_DIR" + ensure_sparse_checkout } move_old_agent_dir() { @@ -96,6 +104,18 @@ move_old_agent_dir() { log "Moving existing backend .env to agent .env..." mv "$OLD_AGENT_DIR/.env" "$ENV_FILE" fi + + if [ "$INSTALL_DIR" != "$LEGACY_INSTALL_DIR" ] && [ ! -f "$ENV_FILE" ]; then + if [ -f "$LEGACY_AGENT_DIR/.env" ]; then + log "Migrating legacy agent .env from $LEGACY_AGENT_DIR..." + cp "$LEGACY_AGENT_DIR/.env" "$ENV_FILE" + chmod 600 "$ENV_FILE" + elif [ -f "$LEGACY_BACKEND_DIR/.env" ]; then + log "Migrating legacy backend .env from $LEGACY_BACKEND_DIR..." + cp "$LEGACY_BACKEND_DIR/.env" "$ENV_FILE" + chmod 600 "$ENV_FILE" + fi + fi } ensure_env_file() { @@ -201,6 +221,8 @@ install_dependencies() { install_systemd_service() { install -d -m 700 "$STATE_DIR" install -d -m 700 "$CONFIG_DIR" + install -m 755 "$INSTALL_DIR/deploy/incubator.sh" /usr/local/bin/incubator + rm -f /usr/local/bin/incus-backup-agentctl tmp_service="$SERVICE_FILE.tmp" awk -v agent_dir="$AGENT_DIR" ' /^WorkingDirectory=/ { @@ -232,6 +254,7 @@ main() { if [ "$(id -u)" -ne 0 ]; then exec sudo \ INSTALL_DIR="$INSTALL_DIR" \ + LEGACY_INSTALL_DIR="$LEGACY_INSTALL_DIR" \ INCUS_BACKUP_REPO_URL="$REPO_URL" \ INCUS_BACKUP_BRANCH="$BRANCH" \ STATE_DIR="$STATE_DIR" \ diff --git a/deploy/systemd/incus-backup-agent.service b/deploy/systemd/incus-backup-agent.service index 0fcc56c..28be01c 100644 --- a/deploy/systemd/incus-backup-agent.service +++ b/deploy/systemd/incus-backup-agent.service @@ -5,7 +5,7 @@ Wants=network-online.target [Service] Type=simple -WorkingDirectory=/opt/incus-backup-ui/agent +WorkingDirectory=/opt/incubator/agent Environment=NODE_ENV=production ExecStart=/usr/bin/npm start Restart=on-failure diff --git a/docs/deployment.md b/docs/deployment.md index 7d60b43..d76aeed 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -19,7 +19,8 @@ curl -fsSL https://forgejo.digital-droplets.de/philschlo/incus-backup-ui/raw/bra Useful installer environment variables: -- `INSTALL_DIR`: install location, default `/opt/incus-backup-ui`. +- `INSTALL_DIR`: install location, default `/opt/incubator`. +- `LEGACY_INSTALL_DIR`: old install location used for `.env` migration, default `/opt/incus-backup-ui`. - `INCUS_BACKUP_REPO_URL`: Git repository URL. - `INCUS_BACKUP_BRANCH`: Git branch, default `main`. - `API_TOKEN`: explicit agent token. If omitted, the installer generates one and prints it once. @@ -71,6 +72,22 @@ sudo systemctl enable --now incus-backup-agent sudo journalctl -u incus-backup-agent -f ``` +The installer uses Git sparse-checkout and only checks out agent-related paths: `agent/` and `deploy/`. +When upgrading from the old default path, it copies an existing `/opt/incus-backup-ui/agent/.env` or `/opt/incus-backup-ui/backend/.env` into `/opt/incubator/agent/.env` if the new env file does not exist yet. + +The installer also installs a small operational CLI: + +```bash +sudo incubator update +sudo incubator status +sudo incubator logs +sudo incubator doctor +sudo incubator restart +sudo incubator env +``` + +Use `update` for normal agent updates. It pulls the configured branch, installs dependencies, refreshes the systemd unit, and restarts the agent. + ## Management API Run this on the management server: diff --git a/docs/issues.md b/docs/issues.md index 0b73db8..5b4b1de 100644 --- a/docs/issues.md +++ b/docs/issues.md @@ -26,11 +26,12 @@ Known caveat: active agent jobs are persisted, but deeper cleanup recovery for p 1. Harden backup verification and add tests for stream/pipeline failure cases. 2. Validate the staged restore workflow on a disposable Incus VM, including rollback scenarios. -3. Add cleanup and visibility for pre-restore/failed-restore ZVOLs. -4. Improve agent crash cleanup for partially changed ZFS/Incus resources. -5. Surface detailed node health diagnostics in the UI. -6. Keep the root-running agent tightly network-restricted. -7. Add automated tests and CI. +3. Evaluate true incremental ZFS-send based backups for large VM disks. +4. Add cleanup and visibility for pre-restore/failed-restore ZVOLs. +5. Improve agent crash cleanup for partially changed ZFS/Incus resources. +6. Surface detailed node health diagnostics in the UI. +7. Keep the root-running agent tightly network-restricted. +8. Add automated tests and CI. ## P0 - Production Blockers @@ -149,7 +150,31 @@ Acceptance criteria: ## P1 - High Priority -### 6. Add Automated Tests and CI +### 6. Evaluate True Incremental VM Backups + +Status: open. + +Current behavior: VM backups stream the full ZVOL as `vm.raw` through `restic backup --stdin`. Restic deduplicates storage, but the agent still has to read, chunk, and hash the complete virtual disk every run. A VM with no changed files can therefore still take a long time. + +Goal: reduce backup duration for large mostly-unchanged VMs by reading only changed ZFS blocks after the first full backup. + +Tasks: + +- [ ] Evaluate ZFS snapshot-chain based incremental backups with `zfs send -i previous current`. +- [ ] Design snapshot naming and retention so required incremental bases are not deleted too early. +- [ ] Define restore behavior for full plus incremental send chains. +- [ ] Decide whether incremental streams should be stored in Restic, object storage directly, or another repository layout. +- [ ] Define compatibility behavior for existing `.raw` Restic snapshots. +- [ ] Compare operational tradeoffs: faster backups versus more complex restore and retention. +- [ ] Add UI wording that distinguishes `Processed` bytes from data actually uploaded/stored. + +Acceptance criteria: + +- [ ] A second backup of an unchanged large VM does not need to read the full ZVOL. +- [ ] Restore can reconstruct a VM from the chosen full/incremental chain. +- [ ] Retention cannot delete an incremental base required for restore. + +### 7. Add Automated Tests and CI Status: open. @@ -170,7 +195,7 @@ Acceptance criteria: - [ ] Simulated restore failures leave the original ZVOL name restored in the command sequence. - [ ] Agent restart behavior is covered by tests. -### 7. Improve Agent Crash Cleanup and Resource Recovery +### 8. Improve Agent Crash Cleanup and Resource Recovery Status: open. @@ -191,7 +216,7 @@ Acceptance criteria: - [ ] Crashing after restore swap does not automatically destroy rollback copies. - [ ] Management can show cleanup-required states. -### 8. Improve Health Diagnostics in the UI +### 9. Improve Health Diagnostics in the UI Status: partially done on API, open in UI. @@ -211,7 +236,7 @@ Acceptance criteria: - [ ] Wrong Restic credentials are visible as a Restic health failure. - [ ] TLS or connectivity failures are distinguishable from degraded agent health. -### 9. Fix Snapshot Device Visibility Race +### 10. Fix Snapshot Device Visibility Race Status: open. @@ -227,7 +252,7 @@ Acceptance criteria: - [ ] Slow device-node creation does not fail randomly. -### 10. Make Snapshot-ID Prefix Matching Unambiguous +### 11. Make Snapshot-ID Prefix Matching Unambiguous Status: open. @@ -243,7 +268,7 @@ Acceptance criteria: - [ ] Ambiguous snapshot prefixes cannot restore the wrong snapshot. -### 11. Settings and Environment Hardening +### 12. Settings and Environment Hardening Status: partially done. @@ -262,7 +287,7 @@ Acceptance criteria: - [ ] Secret settings cannot be exfiltrated through the UI/API. - [ ] Writing `.env` cannot create shell-expansion surprises. -### 12. Add RBAC +### 13. Add RBAC Status: open. @@ -280,7 +305,7 @@ Acceptance criteria: - [ ] Restore is admin-only. - [ ] Viewer cannot trigger backup, restore, node edits, or settings writes. -### 13. Session and Auth Cleanup +### 14. Session and Auth Cleanup Status: partially done. @@ -298,7 +323,7 @@ Acceptance criteria: - [ ] Expired sessions do not accumulate unbounded in SQLite. - [ ] Session IDs are rotated after login. -### 14. Container Restore Decision +### 15. Container Restore Decision Status: open. @@ -316,7 +341,7 @@ Acceptance criteria: ## P2 - Product and Operations -### 15. Add Per-VM Backup Policy +### 16. Add Per-VM Backup Policy Status: partially done. @@ -337,7 +362,7 @@ Acceptance criteria: - [ ] Two VMs on the same node can have different retention policies. - [ ] Disabled policies do not trigger backups. -### 16. Add Failure Notifications +### 17. Add Failure Notifications Status: open. @@ -356,7 +381,7 @@ Acceptance criteria: - [ ] A test notification can be triggered from the UI. - [ ] Notification failures are visible in Operations or audit logs. -### 17. Implement Real Snapshot File Browsing +### 18. Implement Real Snapshot File Browsing Status: open. @@ -377,7 +402,7 @@ Acceptance criteria: - [ ] Mounted/temporary resources are cleaned up after use. - [ ] Unsupported or unsafe disk images fail with a clear error. -### 18. Add Version Reporting +### 19. Add Version Reporting Status: open. @@ -395,7 +420,7 @@ Acceptance criteria: - [ ] Management can identify incompatible agents. - [ ] Health output includes version information. -### 19. Containerized Management and UI Deployment +### 20. Containerized Management and UI Deployment Status: open. @@ -418,7 +443,7 @@ Acceptance criteria: - [ ] Cookie login works behind HTTPS. - [ ] Management can connect to HTTPS node-agents using the configured CA file. -### 20. Backup Scheduling and Quotas +### 21. Backup Scheduling and Quotas Status: open. @@ -436,7 +461,7 @@ Acceptance criteria: ## P3 - Cleanup and Refactoring -### 21. Systemd and Deployment Cleanup +### 22. Systemd and Deployment Cleanup Tasks: @@ -445,7 +470,7 @@ Tasks: - [ ] Make `SCHEDULES_PATH` explicitly configurable. - [ ] Installer should warn on insecure agent exposure. -### 22. Data Integrity and Schema Cleanup +### 23. Data Integrity and Schema Cleanup Tasks: @@ -454,7 +479,7 @@ Tasks: - [ ] Replace ad-hoc `addColumnIfMissing` with schema versioning and migrations. - [ ] Stream Restic `ls` instead of loading all output in RAM. -### 23. Code Cleanup +### 24. Code Cleanup Tasks: @@ -481,6 +506,7 @@ Tasks: - [x] Staged VM restore implemented. - [x] Node-agent renamed from `backend/` to `agent/`. - [x] Agent installer script added. +- [x] Agent update/diagnostic CLI added as `incubator`. - [x] Backup UI ETA/rate/bytes display added. ## OSS Release Requirements diff --git a/frontend/src/components/JobStatusPanel.jsx b/frontend/src/components/JobStatusPanel.jsx index dcc3235..967fda7 100644 --- a/frontend/src/components/JobStatusPanel.jsx +++ b/frontend/src/components/JobStatusPanel.jsx @@ -15,7 +15,7 @@ export function JobStatusPanel({ job }) { - + @@ -33,7 +33,7 @@ export function JobStatusPanel({ job }) { {job ? (
- +
@@ -77,7 +77,7 @@ function transferStats(job) { const remainingSeconds = bytesPerSecond && remainingBytes ? remainingBytes / bytesPerSecond : 0; return { - transferred: currentBytes && totalBytes + processed: currentBytes && totalBytes ? `${formatBytes(currentBytes)} / ${formatBytes(totalBytes)}` : currentBytes ? formatBytes(currentBytes)