changed to incubator

added new install script
updated security
updated ETA

README.md

@@ -23,6 +23,15 @@ Set `API_TOKEN` in `agent/.env`; the management server uses that token when call
 
 The agent persists recent jobs in SQLite through `AGENT_DATABASE_PATH` (default: `./agent.sqlite`). If the agent restarts while a job is `queued` or `running`, that job is marked `failed` on startup so management can poll a final state and stale VM locks are not kept.
 
+The installer clones only the agent-related repository paths and installs `incubator` for updates and diagnostics:
+
+```bash
+sudo incubator update
+sudo incubator status
+sudo incubator logs
+sudo incubator doctor
+```
+
 The agent can serve HTTPS directly for private networks:
 
 ```env

agent/.env.example

@@ -7,6 +7,7 @@ RESTIC_KEEP_HOURLY=0
 RESTIC_KEEP_DAILY=7
 RESTIC_KEEP_WEEKLY=0
 RESTIC_KEEP_MONTHLY=0
+RESTIC_RETRY_LOCK="5m"
 HOST="0.0.0.0"
 PORT=3000
 API_TOKEN="change-me-to-at-least-32-characters"

agent/src/config.js

@@ -34,6 +34,7 @@ export const config = {
     RESTIC_REPOSITORY: process.env.RESTIC_REPOSITORY || '',
     RESTIC_PASSWORD: process.env.RESTIC_PASSWORD || '',
   },
+  resticRetryLock: process.env.RESTIC_RETRY_LOCK || '5m',
   retention: {
     keepHourly: Number(process.env.RESTIC_KEEP_HOURLY || 0),
     keepDaily: Number(process.env.RESTIC_KEEP_DAILY || 7),
@@ -60,6 +61,7 @@ export const editableEnv = [
   { key: 'RESTIC_KEEP_DAILY', label: 'Keep daily snapshots', required: false, secret: false },
   { key: 'RESTIC_KEEP_WEEKLY', label: 'Keep weekly snapshots', required: false, secret: false },
   { key: 'RESTIC_KEEP_MONTHLY', label: 'Keep monthly snapshots', required: false, secret: false },
+  { key: 'RESTIC_RETRY_LOCK', label: 'Restic lock retry duration', required: false, secret: false },
   { key: 'PORT', label: 'API port', required: false, secret: false },
   { key: 'HOST', label: 'API bind host', required: false, secret: false },
   { key: 'API_TOKEN', label: 'API token', required: true, secret: true },
@@ -161,6 +163,7 @@ function applyRuntimeEnv(values) {
   config.resticEnv.AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY || '';
   config.resticEnv.RESTIC_REPOSITORY = process.env.RESTIC_REPOSITORY || '';
   config.resticEnv.RESTIC_PASSWORD = process.env.RESTIC_PASSWORD || '';
+  config.resticRetryLock = process.env.RESTIC_RETRY_LOCK || '5m';
   config.retention.keepHourly = Number(process.env.RESTIC_KEEP_HOURLY || 0);
   config.retention.keepDaily = Number(process.env.RESTIC_KEEP_DAILY || 7);
   config.retention.keepWeekly = Number(process.env.RESTIC_KEEP_WEEKLY || 0);

agent/src/executor.js

@@ -1,6 +1,6 @@
 import { spawn } from 'node:child_process';
 import { PassThrough, Transform } from 'node:stream';
-import { resticProcessEnv } from './config.js';
+import { config, resticProcessEnv } from './config.js';
 
 export class CommandError extends Error {
   constructor(message, result) {
@@ -68,14 +68,14 @@ export function spawnCommand(command, args = [], options = {}) {
 }
 
 export function runRestic(args, options = {}) {
-  return spawnCommand('restic', args, {
+  return spawnCommand('restic', resticArgs(args, options), {
     ...options,
     env: resticProcessEnv(),
   });
 }
 
 export async function resticSnapshotFileSize(snapshotId, filename) {
-  const result = await runRestic(['ls', '--json', snapshotId]);
+  const result = await runRestic(['--no-lock', 'ls', '--json', snapshotId], { retryLock: false });
   const wanted = `/${filename}`;
   for (const line of result.stdout.split('\n')) {
     if (!line.trim()) continue;
@@ -89,6 +89,12 @@ export async function resticSnapshotFileSize(snapshotId, filename) {
   throw error;
 }
 
+export function resticArgs(args, options = {}) {
+  if (options.retryLock === false || !config.resticRetryLock) return args;
+  if (args.includes('--no-lock') || args.includes('--retry-lock')) return args;
+  return ['--retry-lock', config.resticRetryLock, ...args];
+}
+
 export function createProgressStream(totalBytes, onProgress) {
   let currentBytes = 0;
   return new Transform({

agent/src/routes/backup.js

@@ -6,7 +6,7 @@ import path from 'node:path';
 import { finished } from 'node:stream/promises';
 import { setTimeout as delay } from 'node:timers/promises';
 import { config } from '../config.js';
-import { createProgressStream, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js';
+import { createProgressStream, resticArgs, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js';
 import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js';
 import { validateVmExists } from '../validators.js';
 
@@ -70,7 +70,7 @@ export async function runBackupJob(job, instance = null) {
       });
     });
     try {
-      const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine'], {
+      const result = await spawnCommand('restic', resticArgs(['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine']), {
         env: { ...process.env, ...config.resticEnv },
         input: snapshotStream.pipe(progressStream),
         log: (line) => appendJobLog(job, line),
@@ -140,7 +140,7 @@ async function runContainerBackupJob(job) {
     });
     let resticOk = false;
     try {
-      const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container'], {
+      const result = await spawnCommand('restic', resticArgs(['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container']), {
         env: { ...process.env, ...config.resticEnv },
         input: zfs.stdout.pipe(progressStream),
         log: (line) => appendJobLog(job, line),

deploy/install-agent.sh

@@ -4,11 +4,14 @@ set -eu
 SERVICE_NAME="incus-backup-agent"
 DEFAULT_REPO_URL="https://forgejo.digital-droplets.de/philschlo/incus-backup-ui.git"
 
-INSTALL_DIR="${INSTALL_DIR:-/opt/incus-backup-ui}"
+INSTALL_DIR="${INSTALL_DIR:-/opt/incubator}"
+LEGACY_INSTALL_DIR="${LEGACY_INSTALL_DIR:-/opt/incus-backup-ui}"
 REPO_URL="${INCUS_BACKUP_REPO_URL:-$DEFAULT_REPO_URL}"
 BRANCH="${INCUS_BACKUP_BRANCH:-main}"
 AGENT_DIR="$INSTALL_DIR/agent"
 OLD_AGENT_DIR="$INSTALL_DIR/backend"
+LEGACY_AGENT_DIR="$LEGACY_INSTALL_DIR/agent"
+LEGACY_BACKEND_DIR="$LEGACY_INSTALL_DIR/backend"
 STATE_DIR="${STATE_DIR:-/var/lib/incus-backup-agent}"
 CONFIG_DIR="${CONFIG_DIR:-/etc/incus-backup-agent}"
 ENV_FILE="$AGENT_DIR/.env"
@@ -58,21 +61,25 @@ copy_repo_from_script_location() {
   fi
   command -v rsync >/dev/null 2>&1 || return 1
   rsync -a --delete \
-    --exclude '.git' \
     --exclude 'agent/.env' \
     --exclude 'agent/node_modules' \
-    --exclude 'management/node_modules' \
+    "$repo_dir/agent" "$repo_dir/deploy" "$INSTALL_DIR/"
-    --exclude 'frontend/node_modules' \
+}
-    "$repo_dir/" "$INSTALL_DIR/"
+
+ensure_sparse_checkout() {
+  git -C "$INSTALL_DIR" sparse-checkout init --cone
+  git -C "$INSTALL_DIR" sparse-checkout set agent deploy
 }
 
 checkout_or_update_repo() {
   need_cmd git
   if [ -d "$INSTALL_DIR/.git" ]; then
     log "Updating $INSTALL_DIR from Git..."
+    ensure_sparse_checkout
     git -C "$INSTALL_DIR" fetch --prune origin "$BRANCH"
     git -C "$INSTALL_DIR" checkout "$BRANCH"
     git -C "$INSTALL_DIR" pull --ff-only origin "$BRANCH"
+    ensure_sparse_checkout
     return
   fi
 
@@ -83,7 +90,8 @@ checkout_or_update_repo() {
 
   log "Cloning $REPO_URL to $INSTALL_DIR..."
   mkdir -p "$(dirname "$INSTALL_DIR")"
-  git clone --branch "$BRANCH" "$REPO_URL" "$INSTALL_DIR"
+  git clone --filter=blob:none --sparse --branch "$BRANCH" "$REPO_URL" "$INSTALL_DIR"
+  ensure_sparse_checkout
 }
 
 move_old_agent_dir() {
@@ -96,6 +104,18 @@ move_old_agent_dir() {
     log "Moving existing backend .env to agent .env..."
     mv "$OLD_AGENT_DIR/.env" "$ENV_FILE"
   fi
+
+  if [ "$INSTALL_DIR" != "$LEGACY_INSTALL_DIR" ] && [ ! -f "$ENV_FILE" ]; then
+    if [ -f "$LEGACY_AGENT_DIR/.env" ]; then
+      log "Migrating legacy agent .env from $LEGACY_AGENT_DIR..."
+      cp "$LEGACY_AGENT_DIR/.env" "$ENV_FILE"
+      chmod 600 "$ENV_FILE"
+    elif [ -f "$LEGACY_BACKEND_DIR/.env" ]; then
+      log "Migrating legacy backend .env from $LEGACY_BACKEND_DIR..."
+      cp "$LEGACY_BACKEND_DIR/.env" "$ENV_FILE"
+      chmod 600 "$ENV_FILE"
+    fi
+  fi
 }
 
 ensure_env_file() {
@@ -201,6 +221,8 @@ install_dependencies() {
 install_systemd_service() {
   install -d -m 700 "$STATE_DIR"
   install -d -m 700 "$CONFIG_DIR"
+  install -m 755 "$INSTALL_DIR/deploy/incubator.sh" /usr/local/bin/incubator
+  rm -f /usr/local/bin/incus-backup-agentctl
   tmp_service="$SERVICE_FILE.tmp"
   awk -v agent_dir="$AGENT_DIR" '
     /^WorkingDirectory=/ {
@@ -232,6 +254,7 @@ main() {
   if [ "$(id -u)" -ne 0 ]; then
     exec sudo \
       INSTALL_DIR="$INSTALL_DIR" \
+      LEGACY_INSTALL_DIR="$LEGACY_INSTALL_DIR" \
       INCUS_BACKUP_REPO_URL="$REPO_URL" \
       INCUS_BACKUP_BRANCH="$BRANCH" \
       STATE_DIR="$STATE_DIR" \

deploy/systemd/incus-backup-agent.service

@@ -5,7 +5,7 @@ Wants=network-online.target
 
 [Service]
 Type=simple
-WorkingDirectory=/opt/incus-backup-ui/agent
+WorkingDirectory=/opt/incubator/agent
 Environment=NODE_ENV=production
 ExecStart=/usr/bin/npm start
 Restart=on-failure

docs/deployment.md

@@ -19,7 +19,8 @@ curl -fsSL https://forgejo.digital-droplets.de/philschlo/incus-backup-ui/raw/bra
 
 Useful installer environment variables:
 
-- `INSTALL_DIR`: install location, default `/opt/incus-backup-ui`.
+- `INSTALL_DIR`: install location, default `/opt/incubator`.
+- `LEGACY_INSTALL_DIR`: old install location used for `.env` migration, default `/opt/incus-backup-ui`.
 - `INCUS_BACKUP_REPO_URL`: Git repository URL.
 - `INCUS_BACKUP_BRANCH`: Git branch, default `main`.
 - `API_TOKEN`: explicit agent token. If omitted, the installer generates one and prints it once.
@@ -71,6 +72,22 @@ sudo systemctl enable --now incus-backup-agent
 sudo journalctl -u incus-backup-agent -f
 ```
 
+The installer uses Git sparse-checkout and only checks out agent-related paths: `agent/` and `deploy/`.
+When upgrading from the old default path, it copies an existing `/opt/incus-backup-ui/agent/.env` or `/opt/incus-backup-ui/backend/.env` into `/opt/incubator/agent/.env` if the new env file does not exist yet.
+
+The installer also installs a small operational CLI:
+
+```bash
+sudo incubator update
+sudo incubator status
+sudo incubator logs
+sudo incubator doctor
+sudo incubator restart
+sudo incubator env
+```
+
+Use `update` for normal agent updates. It pulls the configured branch, installs dependencies, refreshes the systemd unit, and restarts the agent.
+
 ## Management API
 
 Run this on the management server:

docs/issues.md

@@ -26,11 +26,12 @@ Known caveat: active agent jobs are persisted, but deeper cleanup recovery for p
 
 1. Harden backup verification and add tests for stream/pipeline failure cases.
 2. Validate the staged restore workflow on a disposable Incus VM, including rollback scenarios.
-3. Add cleanup and visibility for pre-restore/failed-restore ZVOLs.
+3. Evaluate true incremental ZFS-send based backups for large VM disks.
-4. Improve agent crash cleanup for partially changed ZFS/Incus resources.
+4. Add cleanup and visibility for pre-restore/failed-restore ZVOLs.
-5. Surface detailed node health diagnostics in the UI.
+5. Improve agent crash cleanup for partially changed ZFS/Incus resources.
-6. Keep the root-running agent tightly network-restricted.
+6. Surface detailed node health diagnostics in the UI.
-7. Add automated tests and CI.
+7. Keep the root-running agent tightly network-restricted.
+8. Add automated tests and CI.
 
 ## P0 - Production Blockers
 
@@ -149,7 +150,31 @@ Acceptance criteria:
 
 ## P1 - High Priority
 
-### 6. Add Automated Tests and CI
+### 6. Evaluate True Incremental VM Backups
+
+Status: open.
+
+Current behavior: VM backups stream the full ZVOL as `vm.raw` through `restic backup --stdin`. Restic deduplicates storage, but the agent still has to read, chunk, and hash the complete virtual disk every run. A VM with no changed files can therefore still take a long time.
+
+Goal: reduce backup duration for large mostly-unchanged VMs by reading only changed ZFS blocks after the first full backup.
+
+Tasks:
+
+- [ ] Evaluate ZFS snapshot-chain based incremental backups with `zfs send -i previous current`.
+- [ ] Design snapshot naming and retention so required incremental bases are not deleted too early.
+- [ ] Define restore behavior for full plus incremental send chains.
+- [ ] Decide whether incremental streams should be stored in Restic, object storage directly, or another repository layout.
+- [ ] Define compatibility behavior for existing `.raw` Restic snapshots.
+- [ ] Compare operational tradeoffs: faster backups versus more complex restore and retention.
+- [ ] Add UI wording that distinguishes `Processed` bytes from data actually uploaded/stored.
+
+Acceptance criteria:
+
+- [ ] A second backup of an unchanged large VM does not need to read the full ZVOL.
+- [ ] Restore can reconstruct a VM from the chosen full/incremental chain.
+- [ ] Retention cannot delete an incremental base required for restore.
+
+### 7. Add Automated Tests and CI
 
 Status: open.
 
@@ -170,7 +195,7 @@ Acceptance criteria:
 - [ ] Simulated restore failures leave the original ZVOL name restored in the command sequence.
 - [ ] Agent restart behavior is covered by tests.
 
-### 7. Improve Agent Crash Cleanup and Resource Recovery
+### 8. Improve Agent Crash Cleanup and Resource Recovery
 
 Status: open.
 
@@ -191,7 +216,7 @@ Acceptance criteria:
 - [ ] Crashing after restore swap does not automatically destroy rollback copies.
 - [ ] Management can show cleanup-required states.
 
-### 8. Improve Health Diagnostics in the UI
+### 9. Improve Health Diagnostics in the UI
 
 Status: partially done on API, open in UI.
 
@@ -211,7 +236,7 @@ Acceptance criteria:
 - [ ] Wrong Restic credentials are visible as a Restic health failure.
 - [ ] TLS or connectivity failures are distinguishable from degraded agent health.
 
-### 9. Fix Snapshot Device Visibility Race
+### 10. Fix Snapshot Device Visibility Race
 
 Status: open.
 
@@ -227,7 +252,7 @@ Acceptance criteria:
 
 - [ ] Slow device-node creation does not fail randomly.
 
-### 10. Make Snapshot-ID Prefix Matching Unambiguous
+### 11. Make Snapshot-ID Prefix Matching Unambiguous
 
 Status: open.
 
@@ -243,7 +268,7 @@ Acceptance criteria:
 
 - [ ] Ambiguous snapshot prefixes cannot restore the wrong snapshot.
 
-### 11. Settings and Environment Hardening
+### 12. Settings and Environment Hardening
 
 Status: partially done.
 
@@ -262,7 +287,7 @@ Acceptance criteria:
 - [ ] Secret settings cannot be exfiltrated through the UI/API.
 - [ ] Writing `.env` cannot create shell-expansion surprises.
 
-### 12. Add RBAC
+### 13. Add RBAC
 
 Status: open.
 
@@ -280,7 +305,7 @@ Acceptance criteria:
 - [ ] Restore is admin-only.
 - [ ] Viewer cannot trigger backup, restore, node edits, or settings writes.
 
-### 13. Session and Auth Cleanup
+### 14. Session and Auth Cleanup
 
 Status: partially done.
 
@@ -298,7 +323,7 @@ Acceptance criteria:
 - [ ] Expired sessions do not accumulate unbounded in SQLite.
 - [ ] Session IDs are rotated after login.
 
-### 14. Container Restore Decision
+### 15. Container Restore Decision
 
 Status: open.
 
@@ -316,7 +341,7 @@ Acceptance criteria:
 
 ## P2 - Product and Operations
 
-### 15. Add Per-VM Backup Policy
+### 16. Add Per-VM Backup Policy
 
 Status: partially done.
 
@@ -337,7 +362,7 @@ Acceptance criteria:
 - [ ] Two VMs on the same node can have different retention policies.
 - [ ] Disabled policies do not trigger backups.
 
-### 16. Add Failure Notifications
+### 17. Add Failure Notifications
 
 Status: open.
 
@@ -356,7 +381,7 @@ Acceptance criteria:
 - [ ] A test notification can be triggered from the UI.
 - [ ] Notification failures are visible in Operations or audit logs.
 
-### 17. Implement Real Snapshot File Browsing
+### 18. Implement Real Snapshot File Browsing
 
 Status: open.
 
@@ -377,7 +402,7 @@ Acceptance criteria:
 - [ ] Mounted/temporary resources are cleaned up after use.
 - [ ] Unsupported or unsafe disk images fail with a clear error.
 
-### 18. Add Version Reporting
+### 19. Add Version Reporting
 
 Status: open.
 
@@ -395,7 +420,7 @@ Acceptance criteria:
 - [ ] Management can identify incompatible agents.
 - [ ] Health output includes version information.
 
-### 19. Containerized Management and UI Deployment
+### 20. Containerized Management and UI Deployment
 
 Status: open.
 
@@ -418,7 +443,7 @@ Acceptance criteria:
 - [ ] Cookie login works behind HTTPS.
 - [ ] Management can connect to HTTPS node-agents using the configured CA file.
 
-### 20. Backup Scheduling and Quotas
+### 21. Backup Scheduling and Quotas
 
 Status: open.
 
@@ -436,7 +461,7 @@ Acceptance criteria:
 
 ## P3 - Cleanup and Refactoring
 
-### 21. Systemd and Deployment Cleanup
+### 22. Systemd and Deployment Cleanup
 
 Tasks:
 
@@ -445,7 +470,7 @@ Tasks:
 - [ ] Make `SCHEDULES_PATH` explicitly configurable.
 - [ ] Installer should warn on insecure agent exposure.
 
-### 22. Data Integrity and Schema Cleanup
+### 23. Data Integrity and Schema Cleanup
 
 Tasks:
 
@@ -454,7 +479,7 @@ Tasks:
 - [ ] Replace ad-hoc `addColumnIfMissing` with schema versioning and migrations.
 - [ ] Stream Restic `ls` instead of loading all output in RAM.
 
-### 23. Code Cleanup
+### 24. Code Cleanup
 
 Tasks:
 
@@ -481,6 +506,7 @@ Tasks:
 - [x] Staged VM restore implemented.
 - [x] Node-agent renamed from `backend/` to `agent/`.
 - [x] Agent installer script added.
+- [x] Agent update/diagnostic CLI added as `incubator`.
 - [x] Backup UI ETA/rate/bytes display added.
 
 ## OSS Release Requirements

frontend/src/components/JobStatusPanel.jsx

@@ -15,7 +15,7 @@ export function JobStatusPanel({ job }) {
           <Info label="Type" value={job?.type || '-'} />
           <Info label="Current Step" value={job?.currentStep || 'No active job'} />
           <Info label="Progress" value={job ? `${percent}%` : '-'} />
-          <Info label="Transferred" value={transfer.transferred} />
+          <Info label="Processed" value={transfer.processed} />
           <Info label="Rate" value={transfer.rate} />
           <Info label="ETA" value={transfer.eta} />
           <Info label="Started" value={formatTime(job?.startedAt)} />
@@ -33,7 +33,7 @@ export function JobStatusPanel({ job }) {
             </div>
             {job ? (
               <div className="mt-3 grid gap-2 text-xs text-zinc-500 sm:grid-cols-3">
-                <Metric label="Transferred" value={transfer.transferred} />
+                <Metric label="Processed" value={transfer.processed} />
                 <Metric label="Rate" value={transfer.rate} />
                 <Metric label="ETA" value={transfer.eta} />
               </div>
@@ -77,7 +77,7 @@ function transferStats(job) {
   const remainingSeconds = bytesPerSecond && remainingBytes ? remainingBytes / bytesPerSecond : 0;
 
   return {
-    transferred: currentBytes && totalBytes
+    processed: currentBytes && totalBytes
       ? `${formatBytes(currentBytes)} / ${formatBytes(totalBytes)}`
       : currentBytes
         ? formatBytes(currentBytes)