diff --git a/README.md b/README.md
index 73a40c8..f091b29 100644
--- a/README.md
+++ b/README.md
@@ -23,6 +23,15 @@ Set `API_TOKEN` in `agent/.env`; the management server uses that token when call
The agent persists recent jobs in SQLite through `AGENT_DATABASE_PATH` (default: `./agent.sqlite`). If the agent restarts while a job is `queued` or `running`, that job is marked `failed` on startup so management can poll a final state and stale VM locks are not kept.
+The installer clones only the agent-related repository paths and installs `incubator` for updates and diagnostics:
+
+```bash
+sudo incubator update
+sudo incubator status
+sudo incubator logs
+sudo incubator doctor
+```
+
The agent can serve HTTPS directly for private networks:
```env
diff --git a/agent/.env.example b/agent/.env.example
index a55fc1b..2df8047 100644
--- a/agent/.env.example
+++ b/agent/.env.example
@@ -7,6 +7,7 @@ RESTIC_KEEP_HOURLY=0
RESTIC_KEEP_DAILY=7
RESTIC_KEEP_WEEKLY=0
RESTIC_KEEP_MONTHLY=0
+RESTIC_RETRY_LOCK="5m"
HOST="0.0.0.0"
PORT=3000
API_TOKEN="change-me-to-at-least-32-characters"
diff --git a/agent/src/config.js b/agent/src/config.js
index 5ea39b4..dbc461e 100644
--- a/agent/src/config.js
+++ b/agent/src/config.js
@@ -34,6 +34,7 @@ export const config = {
RESTIC_REPOSITORY: process.env.RESTIC_REPOSITORY || '',
RESTIC_PASSWORD: process.env.RESTIC_PASSWORD || '',
},
+ resticRetryLock: process.env.RESTIC_RETRY_LOCK || '5m',
retention: {
keepHourly: Number(process.env.RESTIC_KEEP_HOURLY || 0),
keepDaily: Number(process.env.RESTIC_KEEP_DAILY || 7),
@@ -60,6 +61,7 @@ export const editableEnv = [
{ key: 'RESTIC_KEEP_DAILY', label: 'Keep daily snapshots', required: false, secret: false },
{ key: 'RESTIC_KEEP_WEEKLY', label: 'Keep weekly snapshots', required: false, secret: false },
{ key: 'RESTIC_KEEP_MONTHLY', label: 'Keep monthly snapshots', required: false, secret: false },
+ { key: 'RESTIC_RETRY_LOCK', label: 'Restic lock retry duration', required: false, secret: false },
{ key: 'PORT', label: 'API port', required: false, secret: false },
{ key: 'HOST', label: 'API bind host', required: false, secret: false },
{ key: 'API_TOKEN', label: 'API token', required: true, secret: true },
@@ -161,6 +163,7 @@ function applyRuntimeEnv(values) {
config.resticEnv.AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY || '';
config.resticEnv.RESTIC_REPOSITORY = process.env.RESTIC_REPOSITORY || '';
config.resticEnv.RESTIC_PASSWORD = process.env.RESTIC_PASSWORD || '';
+ config.resticRetryLock = process.env.RESTIC_RETRY_LOCK || '5m';
config.retention.keepHourly = Number(process.env.RESTIC_KEEP_HOURLY || 0);
config.retention.keepDaily = Number(process.env.RESTIC_KEEP_DAILY || 7);
config.retention.keepWeekly = Number(process.env.RESTIC_KEEP_WEEKLY || 0);
diff --git a/agent/src/executor.js b/agent/src/executor.js
index 2f20389..79a20b9 100644
--- a/agent/src/executor.js
+++ b/agent/src/executor.js
@@ -1,6 +1,6 @@
import { spawn } from 'node:child_process';
import { PassThrough, Transform } from 'node:stream';
-import { resticProcessEnv } from './config.js';
+import { config, resticProcessEnv } from './config.js';
export class CommandError extends Error {
constructor(message, result) {
@@ -68,14 +68,14 @@ export function spawnCommand(command, args = [], options = {}) {
}
export function runRestic(args, options = {}) {
- return spawnCommand('restic', args, {
+ return spawnCommand('restic', resticArgs(args, options), {
...options,
env: resticProcessEnv(),
});
}
export async function resticSnapshotFileSize(snapshotId, filename) {
- const result = await runRestic(['ls', '--json', snapshotId]);
+ const result = await runRestic(['--no-lock', 'ls', '--json', snapshotId], { retryLock: false });
const wanted = `/${filename}`;
for (const line of result.stdout.split('\n')) {
if (!line.trim()) continue;
@@ -89,6 +89,12 @@ export async function resticSnapshotFileSize(snapshotId, filename) {
throw error;
}
+export function resticArgs(args, options = {}) {
+ if (options.retryLock === false || !config.resticRetryLock) return args;
+ if (args.includes('--no-lock') || args.includes('--retry-lock')) return args;
+ return ['--retry-lock', config.resticRetryLock, ...args];
+}
+
export function createProgressStream(totalBytes, onProgress) {
let currentBytes = 0;
return new Transform({
diff --git a/agent/src/routes/backup.js b/agent/src/routes/backup.js
index d8b51fc..1eab046 100644
--- a/agent/src/routes/backup.js
+++ b/agent/src/routes/backup.js
@@ -6,7 +6,7 @@ import path from 'node:path';
import { finished } from 'node:stream/promises';
import { setTimeout as delay } from 'node:timers/promises';
import { config } from '../config.js';
-import { createProgressStream, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js';
+import { createProgressStream, resticArgs, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js';
import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js';
import { validateVmExists } from '../validators.js';
@@ -70,7 +70,7 @@ export async function runBackupJob(job, instance = null) {
});
});
try {
- const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine'], {
+ const result = await spawnCommand('restic', resticArgs(['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine']), {
env: { ...process.env, ...config.resticEnv },
input: snapshotStream.pipe(progressStream),
log: (line) => appendJobLog(job, line),
@@ -140,7 +140,7 @@ async function runContainerBackupJob(job) {
});
let resticOk = false;
try {
- const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container'], {
+ const result = await spawnCommand('restic', resticArgs(['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container']), {
env: { ...process.env, ...config.resticEnv },
input: zfs.stdout.pipe(progressStream),
log: (line) => appendJobLog(job, line),
diff --git a/deploy/install-agent.sh b/deploy/install-agent.sh
index fdf2275..76bd49f 100755
--- a/deploy/install-agent.sh
+++ b/deploy/install-agent.sh
@@ -4,11 +4,14 @@ set -eu
SERVICE_NAME="incus-backup-agent"
DEFAULT_REPO_URL="https://forgejo.digital-droplets.de/philschlo/incus-backup-ui.git"
-INSTALL_DIR="${INSTALL_DIR:-/opt/incus-backup-ui}"
+INSTALL_DIR="${INSTALL_DIR:-/opt/incubator}"
+LEGACY_INSTALL_DIR="${LEGACY_INSTALL_DIR:-/opt/incus-backup-ui}"
REPO_URL="${INCUS_BACKUP_REPO_URL:-$DEFAULT_REPO_URL}"
BRANCH="${INCUS_BACKUP_BRANCH:-main}"
AGENT_DIR="$INSTALL_DIR/agent"
OLD_AGENT_DIR="$INSTALL_DIR/backend"
+LEGACY_AGENT_DIR="$LEGACY_INSTALL_DIR/agent"
+LEGACY_BACKEND_DIR="$LEGACY_INSTALL_DIR/backend"
STATE_DIR="${STATE_DIR:-/var/lib/incus-backup-agent}"
CONFIG_DIR="${CONFIG_DIR:-/etc/incus-backup-agent}"
ENV_FILE="$AGENT_DIR/.env"
@@ -58,21 +61,25 @@ copy_repo_from_script_location() {
fi
command -v rsync >/dev/null 2>&1 || return 1
rsync -a --delete \
- --exclude '.git' \
--exclude 'agent/.env' \
--exclude 'agent/node_modules' \
- --exclude 'management/node_modules' \
- --exclude 'frontend/node_modules' \
- "$repo_dir/" "$INSTALL_DIR/"
+ "$repo_dir/agent" "$repo_dir/deploy" "$INSTALL_DIR/"
+}
+
+ensure_sparse_checkout() {
+ git -C "$INSTALL_DIR" sparse-checkout init --cone
+ git -C "$INSTALL_DIR" sparse-checkout set agent deploy
}
checkout_or_update_repo() {
need_cmd git
if [ -d "$INSTALL_DIR/.git" ]; then
log "Updating $INSTALL_DIR from Git..."
+ ensure_sparse_checkout
git -C "$INSTALL_DIR" fetch --prune origin "$BRANCH"
git -C "$INSTALL_DIR" checkout "$BRANCH"
git -C "$INSTALL_DIR" pull --ff-only origin "$BRANCH"
+ ensure_sparse_checkout
return
fi
@@ -83,7 +90,8 @@ checkout_or_update_repo() {
log "Cloning $REPO_URL to $INSTALL_DIR..."
mkdir -p "$(dirname "$INSTALL_DIR")"
- git clone --branch "$BRANCH" "$REPO_URL" "$INSTALL_DIR"
+ git clone --filter=blob:none --sparse --branch "$BRANCH" "$REPO_URL" "$INSTALL_DIR"
+ ensure_sparse_checkout
}
move_old_agent_dir() {
@@ -96,6 +104,18 @@ move_old_agent_dir() {
log "Moving existing backend .env to agent .env..."
mv "$OLD_AGENT_DIR/.env" "$ENV_FILE"
fi
+
+ if [ "$INSTALL_DIR" != "$LEGACY_INSTALL_DIR" ] && [ ! -f "$ENV_FILE" ]; then
+ if [ -f "$LEGACY_AGENT_DIR/.env" ]; then
+ log "Migrating legacy agent .env from $LEGACY_AGENT_DIR..."
+ cp "$LEGACY_AGENT_DIR/.env" "$ENV_FILE"
+ chmod 600 "$ENV_FILE"
+ elif [ -f "$LEGACY_BACKEND_DIR/.env" ]; then
+ log "Migrating legacy backend .env from $LEGACY_BACKEND_DIR..."
+ cp "$LEGACY_BACKEND_DIR/.env" "$ENV_FILE"
+ chmod 600 "$ENV_FILE"
+ fi
+ fi
}
ensure_env_file() {
@@ -201,6 +221,8 @@ install_dependencies() {
install_systemd_service() {
install -d -m 700 "$STATE_DIR"
install -d -m 700 "$CONFIG_DIR"
+ install -m 755 "$INSTALL_DIR/deploy/incubator.sh" /usr/local/bin/incubator
+ rm -f /usr/local/bin/incus-backup-agentctl
tmp_service="$SERVICE_FILE.tmp"
awk -v agent_dir="$AGENT_DIR" '
/^WorkingDirectory=/ {
@@ -232,6 +254,7 @@ main() {
if [ "$(id -u)" -ne 0 ]; then
exec sudo \
INSTALL_DIR="$INSTALL_DIR" \
+ LEGACY_INSTALL_DIR="$LEGACY_INSTALL_DIR" \
INCUS_BACKUP_REPO_URL="$REPO_URL" \
INCUS_BACKUP_BRANCH="$BRANCH" \
STATE_DIR="$STATE_DIR" \
diff --git a/deploy/systemd/incus-backup-agent.service b/deploy/systemd/incus-backup-agent.service
index 0fcc56c..28be01c 100644
--- a/deploy/systemd/incus-backup-agent.service
+++ b/deploy/systemd/incus-backup-agent.service
@@ -5,7 +5,7 @@ Wants=network-online.target
[Service]
Type=simple
-WorkingDirectory=/opt/incus-backup-ui/agent
+WorkingDirectory=/opt/incubator/agent
Environment=NODE_ENV=production
ExecStart=/usr/bin/npm start
Restart=on-failure
diff --git a/docs/deployment.md b/docs/deployment.md
index 7d60b43..d76aeed 100644
--- a/docs/deployment.md
+++ b/docs/deployment.md
@@ -19,7 +19,8 @@ curl -fsSL https://forgejo.digital-droplets.de/philschlo/incus-backup-ui/raw/bra
Useful installer environment variables:
-- `INSTALL_DIR`: install location, default `/opt/incus-backup-ui`.
+- `INSTALL_DIR`: install location, default `/opt/incubator`.
+- `LEGACY_INSTALL_DIR`: old install location used for `.env` migration, default `/opt/incus-backup-ui`.
- `INCUS_BACKUP_REPO_URL`: Git repository URL.
- `INCUS_BACKUP_BRANCH`: Git branch, default `main`.
- `API_TOKEN`: explicit agent token. If omitted, the installer generates one and prints it once.
@@ -71,6 +72,22 @@ sudo systemctl enable --now incus-backup-agent
sudo journalctl -u incus-backup-agent -f
```
+The installer uses Git sparse-checkout and only checks out agent-related paths: `agent/` and `deploy/`.
+When upgrading from the old default path, it copies an existing `/opt/incus-backup-ui/agent/.env` or `/opt/incus-backup-ui/backend/.env` into `/opt/incubator/agent/.env` if the new env file does not exist yet.
+
+The installer also installs a small operational CLI:
+
+```bash
+sudo incubator update
+sudo incubator status
+sudo incubator logs
+sudo incubator doctor
+sudo incubator restart
+sudo incubator env
+```
+
+Use `update` for normal agent updates. It pulls the configured branch, installs dependencies, refreshes the systemd unit, and restarts the agent.
+
## Management API
Run this on the management server:
diff --git a/docs/issues.md b/docs/issues.md
index 0b73db8..5b4b1de 100644
--- a/docs/issues.md
+++ b/docs/issues.md
@@ -26,11 +26,12 @@ Known caveat: active agent jobs are persisted, but deeper cleanup recovery for p
1. Harden backup verification and add tests for stream/pipeline failure cases.
2. Validate the staged restore workflow on a disposable Incus VM, including rollback scenarios.
-3. Add cleanup and visibility for pre-restore/failed-restore ZVOLs.
-4. Improve agent crash cleanup for partially changed ZFS/Incus resources.
-5. Surface detailed node health diagnostics in the UI.
-6. Keep the root-running agent tightly network-restricted.
-7. Add automated tests and CI.
+3. Evaluate true incremental ZFS-send based backups for large VM disks.
+4. Add cleanup and visibility for pre-restore/failed-restore ZVOLs.
+5. Improve agent crash cleanup for partially changed ZFS/Incus resources.
+6. Surface detailed node health diagnostics in the UI.
+7. Keep the root-running agent tightly network-restricted.
+8. Add automated tests and CI.
## P0 - Production Blockers
@@ -149,7 +150,31 @@ Acceptance criteria:
## P1 - High Priority
-### 6. Add Automated Tests and CI
+### 6. Evaluate True Incremental VM Backups
+
+Status: open.
+
+Current behavior: VM backups stream the full ZVOL as `vm.raw` through `restic backup --stdin`. Restic deduplicates storage, but the agent still has to read, chunk, and hash the complete virtual disk every run. A VM with no changed files can therefore still take a long time.
+
+Goal: reduce backup duration for large mostly-unchanged VMs by reading only changed ZFS blocks after the first full backup.
+
+Tasks:
+
+- [ ] Evaluate ZFS snapshot-chain based incremental backups with `zfs send -i previous current`.
+- [ ] Design snapshot naming and retention so required incremental bases are not deleted too early.
+- [ ] Define restore behavior for full plus incremental send chains.
+- [ ] Decide whether incremental streams should be stored in Restic, object storage directly, or another repository layout.
+- [ ] Define compatibility behavior for existing `.raw` Restic snapshots.
+- [ ] Compare operational tradeoffs: faster backups versus more complex restore and retention.
+- [ ] Add UI wording that distinguishes `Processed` bytes from data actually uploaded/stored.
+
+Acceptance criteria:
+
+- [ ] A second backup of an unchanged large VM does not need to read the full ZVOL.
+- [ ] Restore can reconstruct a VM from the chosen full/incremental chain.
+- [ ] Retention cannot delete an incremental base required for restore.
+
+### 7. Add Automated Tests and CI
Status: open.
@@ -170,7 +195,7 @@ Acceptance criteria:
- [ ] Simulated restore failures leave the original ZVOL name restored in the command sequence.
- [ ] Agent restart behavior is covered by tests.
-### 7. Improve Agent Crash Cleanup and Resource Recovery
+### 8. Improve Agent Crash Cleanup and Resource Recovery
Status: open.
@@ -191,7 +216,7 @@ Acceptance criteria:
- [ ] Crashing after restore swap does not automatically destroy rollback copies.
- [ ] Management can show cleanup-required states.
-### 8. Improve Health Diagnostics in the UI
+### 9. Improve Health Diagnostics in the UI
Status: partially done on API, open in UI.
@@ -211,7 +236,7 @@ Acceptance criteria:
- [ ] Wrong Restic credentials are visible as a Restic health failure.
- [ ] TLS or connectivity failures are distinguishable from degraded agent health.
-### 9. Fix Snapshot Device Visibility Race
+### 10. Fix Snapshot Device Visibility Race
Status: open.
@@ -227,7 +252,7 @@ Acceptance criteria:
- [ ] Slow device-node creation does not fail randomly.
-### 10. Make Snapshot-ID Prefix Matching Unambiguous
+### 11. Make Snapshot-ID Prefix Matching Unambiguous
Status: open.
@@ -243,7 +268,7 @@ Acceptance criteria:
- [ ] Ambiguous snapshot prefixes cannot restore the wrong snapshot.
-### 11. Settings and Environment Hardening
+### 12. Settings and Environment Hardening
Status: partially done.
@@ -262,7 +287,7 @@ Acceptance criteria:
- [ ] Secret settings cannot be exfiltrated through the UI/API.
- [ ] Writing `.env` cannot create shell-expansion surprises.
-### 12. Add RBAC
+### 13. Add RBAC
Status: open.
@@ -280,7 +305,7 @@ Acceptance criteria:
- [ ] Restore is admin-only.
- [ ] Viewer cannot trigger backup, restore, node edits, or settings writes.
-### 13. Session and Auth Cleanup
+### 14. Session and Auth Cleanup
Status: partially done.
@@ -298,7 +323,7 @@ Acceptance criteria:
- [ ] Expired sessions do not accumulate unbounded in SQLite.
- [ ] Session IDs are rotated after login.
-### 14. Container Restore Decision
+### 15. Container Restore Decision
Status: open.
@@ -316,7 +341,7 @@ Acceptance criteria:
## P2 - Product and Operations
-### 15. Add Per-VM Backup Policy
+### 16. Add Per-VM Backup Policy
Status: partially done.
@@ -337,7 +362,7 @@ Acceptance criteria:
- [ ] Two VMs on the same node can have different retention policies.
- [ ] Disabled policies do not trigger backups.
-### 16. Add Failure Notifications
+### 17. Add Failure Notifications
Status: open.
@@ -356,7 +381,7 @@ Acceptance criteria:
- [ ] A test notification can be triggered from the UI.
- [ ] Notification failures are visible in Operations or audit logs.
-### 17. Implement Real Snapshot File Browsing
+### 18. Implement Real Snapshot File Browsing
Status: open.
@@ -377,7 +402,7 @@ Acceptance criteria:
- [ ] Mounted/temporary resources are cleaned up after use.
- [ ] Unsupported or unsafe disk images fail with a clear error.
-### 18. Add Version Reporting
+### 19. Add Version Reporting
Status: open.
@@ -395,7 +420,7 @@ Acceptance criteria:
- [ ] Management can identify incompatible agents.
- [ ] Health output includes version information.
-### 19. Containerized Management and UI Deployment
+### 20. Containerized Management and UI Deployment
Status: open.
@@ -418,7 +443,7 @@ Acceptance criteria:
- [ ] Cookie login works behind HTTPS.
- [ ] Management can connect to HTTPS node-agents using the configured CA file.
-### 20. Backup Scheduling and Quotas
+### 21. Backup Scheduling and Quotas
Status: open.
@@ -436,7 +461,7 @@ Acceptance criteria:
## P3 - Cleanup and Refactoring
-### 21. Systemd and Deployment Cleanup
+### 22. Systemd and Deployment Cleanup
Tasks:
@@ -445,7 +470,7 @@ Tasks:
- [ ] Make `SCHEDULES_PATH` explicitly configurable.
- [ ] Installer should warn on insecure agent exposure.
-### 22. Data Integrity and Schema Cleanup
+### 23. Data Integrity and Schema Cleanup
Tasks:
@@ -454,7 +479,7 @@ Tasks:
- [ ] Replace ad-hoc `addColumnIfMissing` with schema versioning and migrations.
- [ ] Stream Restic `ls` instead of loading all output in RAM.
-### 23. Code Cleanup
+### 24. Code Cleanup
Tasks:
@@ -481,6 +506,7 @@ Tasks:
- [x] Staged VM restore implemented.
- [x] Node-agent renamed from `backend/` to `agent/`.
- [x] Agent installer script added.
+- [x] Agent update/diagnostic CLI added as `incubator`.
- [x] Backup UI ETA/rate/bytes display added.
## OSS Release Requirements
diff --git a/frontend/src/components/JobStatusPanel.jsx b/frontend/src/components/JobStatusPanel.jsx
index dcc3235..967fda7 100644
--- a/frontend/src/components/JobStatusPanel.jsx
+++ b/frontend/src/components/JobStatusPanel.jsx
@@ -15,7 +15,7 @@ export function JobStatusPanel({ job }) {