security hardening

This commit is contained in:
Philipp
2026-05-21 14:09:08 +02:00
parent 3103aba972
commit 0046156e58
16 changed files with 205 additions and 43 deletions
+5 -3
View File
@@ -22,7 +22,7 @@ npm run dev
The node agent must run on every Incus host with permission to access Incus, ZFS, `/dev/zvol`, Restic, and S3 credentials. In production this usually means running it as root or through a tightly scoped service account with the needed privileges.
Set `API_TOKEN` in `backend/.env`; the management server uses that token when calling the agent.
Set `API_TOKEN` in `backend/.env`; the management server uses that token when calling the agent. The token is required and must be at least 32 characters long.
Required commands:
@@ -42,7 +42,9 @@ npm install
npm run dev
```
The management API stores nodes, users, sessions, and central schedules in SQLite. Configure the first admin user through `AUTH_USERNAME` and `AUTH_PASSWORD` before the first start. If no password is configured, the development fallback is `admin`.
The management API stores nodes, users, sessions, and central schedules in SQLite. Configure the first admin user through `AUTH_USERNAME` and `AUTH_PASSWORD` before the first start. Startup fails if the initial password is missing.
Agent URLs must use `https://` by default. For local development only, set `ALLOW_INSECURE_AGENT_HTTP=true` in `management/.env` to permit `http://` node URLs.
The management API uses Node's built-in SQLite module and requires Node.js 22.5 or newer.
@@ -71,7 +73,7 @@ Changing most node-agent values applies to new API calls and jobs immediately. C
## Safety Notes
Restore is intentionally guarded twice: the backend validates the snapshot against the VM, and the UI requires typing the VM name before sending the restore request. Restore jobs are never retried automatically.
Restore is intentionally guarded twice: the backend validates the snapshot against the VM, and the UI requires typing the VM name before sending the restore request. VM restores create a pre-restore ZFS snapshot and roll back to it if writing the disk fails. Restore jobs are never retried automatically.
## Deployment
+1 -1
View File
@@ -8,5 +8,5 @@ RESTIC_KEEP_DAILY=7
RESTIC_KEEP_WEEKLY=0
RESTIC_KEEP_MONTHLY=0
PORT=3000
API_TOKEN=""
API_TOKEN="change-me-to-at-least-32-characters"
ALLOWED_MANAGEMENT_IPS=""
+11 -1
View File
@@ -5,6 +5,7 @@ import path from 'node:path';
dotenv.config();
const envPath = path.resolve(process.cwd(), '.env');
const minApiTokenLength = 32;
export const requiredEnv = [
'AWS_ACCESS_KEY_ID',
@@ -36,6 +37,10 @@ export const config = {
},
};
if (!config.apiToken || config.apiToken.length < minApiTokenLength) {
throw new Error(`API_TOKEN is required and must be at least ${minApiTokenLength} characters long.`);
}
export const editableEnv = [
{ key: 'AWS_ACCESS_KEY_ID', label: 'AWS access key ID', required: true, secret: true },
{ key: 'AWS_SECRET_ACCESS_KEY', label: 'AWS secret access key', required: true, secret: true },
@@ -47,7 +52,7 @@ export const editableEnv = [
{ key: 'RESTIC_KEEP_WEEKLY', label: 'Keep weekly snapshots', required: false, secret: false },
{ key: 'RESTIC_KEEP_MONTHLY', label: 'Keep monthly snapshots', required: false, secret: false },
{ key: 'PORT', label: 'API port', required: false, secret: false },
{ key: 'API_TOKEN', label: 'API token', required: false, secret: true },
{ key: 'API_TOKEN', label: 'API token', required: true, secret: true },
{ key: 'ALLOWED_MANAGEMENT_IPS', label: 'Allowed management IPs', required: false, secret: false },
];
@@ -77,6 +82,11 @@ export async function writeEnvSettings(values) {
for (const [key, value] of Object.entries(values || {})) {
if (!allowedKeys.has(key)) continue;
if (key === 'API_TOKEN' && String(value || '').length < minApiTokenLength) {
const error = new Error(`API_TOKEN must be at least ${minApiTokenLength} characters long.`);
error.status = 400;
throw error;
}
nextValues[key] = String(value ?? '');
}
+15
View File
@@ -74,6 +74,21 @@ export function runRestic(args, options = {}) {
});
}
export async function resticSnapshotFileSize(snapshotId, filename) {
const result = await runRestic(['ls', '--json', snapshotId]);
const wanted = `/${filename}`;
for (const line of result.stdout.split('\n')) {
if (!line.trim()) continue;
const entry = JSON.parse(line);
if (entry.type === 'file' && (entry.path === filename || entry.path === wanted || entry.path?.endsWith(wanted))) {
return Number(entry.size);
}
}
const error = new Error(`Could not verify Restic file size for ${filename} in snapshot ${snapshotId}.`);
error.status = 500;
throw error;
}
export function createProgressStream(totalBytes, onProgress) {
let currentBytes = 0;
return new Transform({
-4
View File
@@ -21,10 +21,6 @@ app.use((req, res, next) => {
res.status(403).json({ error: 'Forbidden management source.' });
return;
}
if (!config.apiToken) {
next();
return;
}
const header = req.get('authorization') || '';
if (header === `Bearer ${config.apiToken}`) {
next();
+45 -3
View File
@@ -6,7 +6,7 @@ import path from 'node:path';
import { finished } from 'node:stream/promises';
import { setTimeout as delay } from 'node:timers/promises';
import { config } from '../config.js';
import { createProgressStream, spawnCommand, runRestic } from '../executor.js';
import { createProgressStream, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js';
import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js';
import { validateVmExists } from '../validators.js';
@@ -59,7 +59,9 @@ export async function runBackupJob(job, instance = null) {
setJobStep(job, 'Streaming block device to Restic');
const snapshotStream = await deviceReadStream(snapshotDevice);
let streamedBytes = 0;
const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
streamedBytes = currentBytes;
setJobProgress(job, {
currentBytes,
totalBytes: bytesTotal,
@@ -68,11 +70,13 @@ export async function runBackupJob(job, instance = null) {
});
});
try {
await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine'], {
const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine'], {
env: { ...process.env, ...config.resticEnv },
input: snapshotStream.pipe(progressStream),
log: (line) => appendJobLog(job, line),
});
const snapshotId = parseResticSnapshotId(result);
await verifyResticFileSize(job, snapshotId, `${job.vmName}.raw`, streamedBytes || totalBytes);
} finally {
if (!snapshotStream.destroyed) {
snapshotStream.destroy();
@@ -124,7 +128,9 @@ async function runContainerBackupJob(job) {
const zfs = spawn('zfs', ['send', snapshot], { stdio: ['ignore', 'pipe', 'pipe'] });
const zfsClosed = waitForProcess(zfs, 'zfs send');
zfs.stderr.on('data', (chunk) => appendJobLog(job, chunk.toString().trimEnd()));
let streamedBytes = 0;
const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
streamedBytes = currentBytes;
setJobProgress(job, {
currentBytes,
totalBytes: bytesTotal,
@@ -134,11 +140,13 @@ async function runContainerBackupJob(job) {
});
let resticOk = false;
try {
await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container'], {
const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container'], {
env: { ...process.env, ...config.resticEnv },
input: zfs.stdout.pipe(progressStream),
log: (line) => appendJobLog(job, line),
});
const snapshotId = parseResticSnapshotId(result);
await verifyResticFileSize(job, snapshotId, `${job.vmName}.zfs`, streamedBytes || totalBytes);
resticOk = true;
} finally {
if (!resticOk && !zfs.killed) zfs.kill('SIGTERM');
@@ -270,6 +278,40 @@ function positiveInteger(value) {
return Number.isFinite(number) && number > 0 ? Math.floor(number) : 0;
}
function parseResticSnapshotId(result) {
const output = `${result.stdout || ''}\n${result.stderr || ''}`;
const match = output.match(/snapshot\s+([0-9a-f]{8,64})\s+saved/i);
if (!match) {
throw new Error('Restic backup completed but no snapshot ID could be parsed.');
}
return match[1];
}
async function verifyResticFileSize(job, snapshotId, filename, expectedBytes) {
if (!expectedBytes) {
appendJobLog(job, `Skipping size verification for ${snapshotId}: expected size is unknown.`);
return;
}
try {
const storedBytes = await resticSnapshotFileSize(snapshotId, filename);
if (storedBytes !== expectedBytes) {
throw new Error(`Backup verification failed for ${snapshotId}: stored ${formatBytes(storedBytes)}, expected ${formatBytes(expectedBytes)}.`);
}
appendJobLog(job, `Verified Restic snapshot ${snapshotId}: ${formatBytes(storedBytes)}.`);
} catch (error) {
appendJobLog(job, error.message);
await forgetFailedSnapshot(job, snapshotId);
throw error;
}
}
async function forgetFailedSnapshot(job, snapshotId) {
appendJobLog(job, `Removing failed Restic snapshot ${snapshotId}.`);
await runRestic(['forget', snapshotId, '--prune'], {
log: (line) => appendJobLog(job, line),
}).catch((error) => appendJobLog(job, `Failed to remove Restic snapshot ${snapshotId}: ${error.message}`));
}
async function cleanupBackup(job, zvol, snapshotName) {
setJobStep(job, 'Running cleanup');
await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], {
+27 -6
View File
@@ -1,6 +1,6 @@
import { Router } from 'express';
import { config } from '../config.js';
import { spawnCommand, streamResticDumpToDd } from '../executor.js';
import { resticSnapshotFileSize, spawnCommand, streamResticDumpToDd } from '../executor.js';
import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js';
import { validateSnapshotForVm, validateVmExists } from '../validators.js';
@@ -34,8 +34,10 @@ restoreRouter.post('/:vmName', async (req, res, next) => {
async function runRestoreJob(job, snapshotId) {
const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
const device = `/dev/zvol/${zvol}`;
const rollbackSnapshot = `${zvol}@pre-restore-${new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}`;
let volmodeDev = false;
let diskWriteOk = false;
let rollbackSnapshotCreated = false;
try {
setJobRunning(job, 'Stopping VM');
@@ -45,19 +47,31 @@ async function runRestoreJob(job, snapshotId) {
log: (line) => appendJobLog(job, line),
});
setJobStep(job, 'Creating pre-restore ZFS snapshot');
setJobProgress(job, { percent: 6 });
await spawnCommand('zfs', ['snapshot', rollbackSnapshot], { log: (line) => appendJobLog(job, line) });
rollbackSnapshotCreated = true;
appendJobLog(job, `Created rollback snapshot ${rollbackSnapshot}`);
setJobStep(job, 'Setting ZFS volume to device mode');
setJobProgress(job, { percent: 8 });
setJobProgress(job, { percent: 10 });
await spawnCommand('zfs', ['set', 'volmode=dev', zvol], { log: (line) => appendJobLog(job, line) });
volmodeDev = true;
setJobStep(job, 'Settling device nodes');
setJobProgress(job, { percent: 12 });
setJobProgress(job, { percent: 14 });
await spawnCommand('udevadm', ['trigger'], { log: (line) => appendJobLog(job, line) });
await spawnCommand('udevadm', ['settle'], { log: (line) => appendJobLog(job, line) });
await new Promise((resolve) => setTimeout(resolve, 2000));
setJobStep(job, 'Writing Restic snapshot to block device');
setJobStep(job, 'Checking restore size');
const totalBytes = await zfsVolumeSize(zvol);
const resticBytes = await resticSnapshotFileSize(snapshotId, `${job.vmName}.raw`);
if (totalBytes && resticBytes && totalBytes !== resticBytes) {
throw new Error(`Restore size mismatch: Restic file is ${formatBytes(resticBytes)}, target volume is ${formatBytes(totalBytes)}.`);
}
setJobStep(job, 'Writing Restic snapshot to block device');
await streamResticDumpToDd(snapshotId, `${job.vmName}.raw`, device, (line) => appendJobLog(job, line), {
totalBytes,
onProgress: ({ currentBytes, totalBytes: bytesTotal, percent }) => {
@@ -80,6 +94,7 @@ async function runRestoreJob(job, snapshotId) {
setJobProgress(job, { percent: 97 });
await spawnCommand('incus', ['start', job.vmName], { log: (line) => appendJobLog(job, line) });
appendJobLog(job, `Pre-restore rollback snapshot kept: ${rollbackSnapshot}`);
finishJob(job, 'success');
} catch (error) {
appendJobLog(job, error.message);
@@ -90,9 +105,15 @@ async function runRestoreJob(job, snapshotId) {
log: (line) => appendJobLog(job, line),
}).catch((cleanupError) => appendJobLog(job, cleanupError.message));
}
if (!diskWriteOk) {
appendJobLog(job, 'VM was not restarted because disk restore did not complete successfully.');
if (rollbackSnapshotCreated && !diskWriteOk) {
setJobStep(job, 'Rolling back failed restore');
await spawnCommand('zfs', ['rollback', '-r', rollbackSnapshot], {
ignoreExitCode: true,
log: (line) => appendJobLog(job, line),
}).catch((cleanupError) => appendJobLog(job, cleanupError.message));
appendJobLog(job, `Rolled back to ${rollbackSnapshot}`);
}
appendJobLog(job, 'VM was not restarted because restore did not complete successfully.');
finishJob(job, 'failed', error);
}
}
+7 -2
View File
@@ -15,11 +15,11 @@ Important `.env` values:
```env
PORT=3000
API_TOKEN="long-random-token"
API_TOKEN="long-random-token-at-least-32-characters"
ALLOWED_MANAGEMENT_IPS="management-server-ip"
```
If `ALLOWED_MANAGEMENT_IPS` is set, the agent only accepts requests from those comma-separated IP addresses.
`API_TOKEN` is required and must be at least 32 characters long. If `ALLOWED_MANAGEMENT_IPS` is set, the agent only accepts requests from those comma-separated IP addresses.
Install systemd service:
@@ -49,8 +49,13 @@ SESSION_SECRET="long-random-secret"
AUTH_USERNAME="admin"
AUTH_PASSWORD="initial-password"
DATABASE_PATH="./management.sqlite"
CORS_ORIGINS="https://backup.example.com"
SESSION_COOKIE_SECURE=true
ALLOW_INSECURE_AGENT_HTTP=false
```
`AUTH_PASSWORD` is required for the first start when the user database is empty. `CORS_ORIGINS` must list the frontend origins that are allowed to use cookie-authenticated API calls. Agent URLs must use `https://`; only set `ALLOW_INSECURE_AGENT_HTTP=true` for local development.
Reset an existing password:
```bash
-16
View File
@@ -6,22 +6,6 @@ export const api = axios.create({
withCredentials: true,
});
const token = window.localStorage.getItem('incusBackupApiToken') || import.meta.env.VITE_API_TOKEN;
if (token) {
api.defaults.headers.common.Authorization = `Bearer ${token}`;
}
export function setApiToken(tokenValue) {
const token = String(tokenValue || '');
if (token) {
window.localStorage.setItem('incusBackupApiToken', token);
api.defaults.headers.common.Authorization = `Bearer ${token}`;
return;
}
window.localStorage.removeItem('incusBackupApiToken');
delete api.defaults.headers.common.Authorization;
}
export function errorMessage(error) {
return error.response?.data?.error || error.message || 'Request failed.';
}
+3
View File
@@ -3,3 +3,6 @@ SESSION_SECRET="change-me"
AUTH_USERNAME="admin"
AUTH_PASSWORD="change-me"
DATABASE_PATH="./management.sqlite"
CORS_ORIGINS="http://localhost:5173"
SESSION_COOKIE_SECURE=false
ALLOW_INSECURE_AGENT_HTTP=true
+8 -2
View File
@@ -1,4 +1,5 @@
import { createSession, deleteSession, getUserBySession, getUserByUsername } from './store.js';
import { config } from './config.js';
import { verifyPassword } from './crypto.js';
const cookieName = 'incus_backup_session';
@@ -34,12 +35,12 @@ export function logout(sessionId) {
export function setSessionCookie(res, session) {
res.setHeader('Set-Cookie', [
`${cookieName}=${session.id}; Path=/; HttpOnly; SameSite=Lax; Expires=${session.expiresAt.toUTCString()}`,
cookieHeader(cookieName, session.id, `Expires=${session.expiresAt.toUTCString()}`),
]);
}
export function clearSessionCookie(res) {
res.setHeader('Set-Cookie', [`${cookieName}=; Path=/; HttpOnly; SameSite=Lax; Max-Age=0`]);
res.setHeader('Set-Cookie', [cookieHeader(cookieName, '', 'Max-Age=0')]);
}
function readCookie(req, name) {
@@ -50,3 +51,8 @@ function readCookie(req, name) {
}
return '';
}
function cookieHeader(name, value, lifetime) {
const secure = config.sessionCookieSecure ? '; Secure' : '';
return `${name}=${encodeURIComponent(value)}; Path=/; HttpOnly; SameSite=Strict${secure}; ${lifetime}`;
}
+8
View File
@@ -9,4 +9,12 @@ export const config = {
authUsername: process.env.AUTH_USERNAME || 'admin',
authPassword: process.env.AUTH_PASSWORD || '',
databasePath: path.resolve(process.cwd(), process.env.DATABASE_PATH || './management.sqlite'),
corsOrigins: (process.env.CORS_ORIGINS || 'http://localhost:5173')
.split(',')
.map((value) => value.trim())
.filter(Boolean),
sessionCookieSecure: process.env.SESSION_COOKIE_SECURE
? process.env.SESSION_COOKIE_SECURE === 'true'
: process.env.NODE_ENV === 'production',
allowInsecureAgentHttp: process.env.ALLOW_INSECURE_AGENT_HTTP === 'true',
};
+4 -2
View File
@@ -80,10 +80,12 @@ export function migrate() {
const existing = db.prepare('SELECT id FROM users LIMIT 1').get();
if (!existing) {
if (!config.authPassword) {
throw new Error('AUTH_PASSWORD is required for initial admin user creation. Set AUTH_PASSWORD before starting management.');
}
const now = new Date().toISOString();
const password = config.authPassword || 'admin';
db.prepare('INSERT INTO users (id, username, password_hash, created_at) VALUES (?, ?, ?, ?)')
.run(`user_${cryptoId()}`, config.authUsername, hashPassword(password), now);
.run(`user_${cryptoId()}`, config.authUsername, hashPassword(config.authPassword), now);
}
}
+10 -1
View File
@@ -15,7 +15,16 @@ migrate();
const app = express();
app.use(cors({ origin: true, credentials: true }));
app.use(cors({
origin(origin, callback) {
if (!origin || config.corsOrigins.includes(origin)) {
callback(null, true);
return;
}
callback(new Error('CORS origin is not allowed.'));
},
credentials: true,
}));
app.use(express.json());
app.use('/api/auth', authRouter);
+47 -1
View File
@@ -4,17 +4,43 @@ import { createAuditEvent } from '../store.js';
export const authRouter = Router();
const loginAttempts = new Map();
const maxAttempts = 5;
const windowMs = 15 * 60 * 1000;
authRouter.get('/session', (req, res) => {
const session = currentSession(req);
res.json({ authenticated: Boolean(session), user: session?.user || null });
});
authRouter.post('/login', (req, res) => {
const result = login(req.body?.username, req.body?.password);
const username = String(req.body?.username || '');
const key = loginAttemptKey(req, username);
const attempt = currentAttempt(key);
if (attempt.count >= maxAttempts) {
createAuditEvent({
action: 'login_blocked',
targetType: 'session',
targetId: username || null,
details: { ip: clientIp(req), reason: 'rate_limit' },
});
res.status(429).json({ error: 'Too many login attempts. Try again later.' });
return;
}
const result = login(username, req.body?.password);
if (!result) {
recordFailedAttempt(key);
createAuditEvent({
action: 'login_failed',
targetType: 'session',
targetId: username || null,
details: { ip: clientIp(req) },
});
res.status(401).json({ error: 'Invalid username or password.' });
return;
}
loginAttempts.delete(key);
setSessionCookie(res, result.session);
createAuditEvent({
user: result.user,
@@ -39,3 +65,23 @@ authRouter.post('/logout', (req, res) => {
clearSessionCookie(res);
res.json({ ok: true });
});
function loginAttemptKey(req, username) {
return `${clientIp(req)}:${String(username || '').toLowerCase()}`;
}
function clientIp(req) {
return String(req.ip || req.socket?.remoteAddress || '').replace(/^::ffff:/, '');
}
function currentAttempt(key) {
const now = Date.now();
const current = loginAttempts.get(key);
if (!current || current.resetAt <= now) return { count: 0, resetAt: now + windowMs };
return current;
}
function recordFailedAttempt(key) {
const attempt = currentAttempt(key);
loginAttempts.set(key, { count: attempt.count + 1, resetAt: attempt.resetAt });
}
+14 -1
View File
@@ -1,5 +1,6 @@
import { Router } from 'express';
import { agentRequest, publicNode } from '../agentClient.js';
import { config } from '../config.js';
import { createAuditEvent, createNode, deleteNode, getNode, listNodes, recordNodeHealth, updateNode } from '../store.js';
export const nodesRouter = Router();
@@ -88,7 +89,7 @@ function validateNodeInput(body, options = {}) {
error.status = 400;
throw error;
}
if (baseUrl) new URL(baseUrl);
if (baseUrl) validateBaseUrl(baseUrl);
return {
...(name !== undefined ? { name } : {}),
@@ -97,3 +98,15 @@ function validateNodeInput(body, options = {}) {
...(values.enabled !== undefined ? { enabled: Boolean(values.enabled) } : {}),
};
}
function validateBaseUrl(baseUrl) {
const url = new URL(baseUrl);
if (url.protocol === 'https:') return;
if (url.protocol === 'http:' && config.allowInsecureAgentHttp) {
console.warn(`ALLOW_INSECURE_AGENT_HTTP=true permits insecure agent URL: ${baseUrl}`);
return;
}
const error = new Error('Node URL must use https://. Set ALLOW_INSECURE_AGENT_HTTP=true only for local development.');
error.status = 400;
throw error;
}