Compare commits

..
Author SHA1 Message Date
Pasha Sviderski da09d22b47 docs: draft post wireguard overlay 2025-07-08 21:08:29 +10:00
180 changed files with 20295 additions and 24718 deletions
-1
View File
@@ -1 +0,0 @@
../AI.md
+1 -3
View File
@@ -37,9 +37,7 @@ jobs:
(echo "go.mod or go.sum has changed. Please run 'go mod tidy' and commit the changes." && exit 1)
- name: Run tests
run: |
make ucind-image
make test
run: make test
timeout-minutes: 10
check-protobuf:
-39
View File
@@ -1,39 +0,0 @@
name: Lint
on:
push:
branches:
- "main"
- "test/**"
- "release/**"
pull_request:
branches:
- main
paths:
- "**.go"
- "go.*"
permissions:
contents: read
jobs:
lint:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up Go
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
with:
go-version: "1.23.2"
- name: golangci-lint
uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0
with:
version: v2.2.2
- name: Format code
run: |
make format
git diff --exit-code ||
(echo "Code is not formatted. Please run 'make format' and commit the changes." && exit 1)
timeout-minutes: 10
-42
View File
@@ -1,42 +0,0 @@
version: "2"
run:
concurrency: 4
tests: true
timeout: 5m
linters:
default: none
enable:
- bodyclose
# - dogsled
- dupl
# - errcheck
- gochecknoinits
- goconst
# - gocritic
# - gocyclo
# - godot
# - gosec
# - govet
- ineffassign
- misspell
- nakedret
# - prealloc
# - revive
# - staticcheck
- unconvert
# - unparam
# - unused
- whitespace
exclusions:
rules:
- path: ^test/e2e
linters:
- goconst # constants here add no value, so we skip goconst only for test/e2e.
formatters:
enable:
- gofumpt
- goimports
-8
View File
@@ -6,14 +6,6 @@ backend = "core:go"
"go1.23.10.darwin-arm64.tar.gz" = "sha256:25c64bfa8a8fd8e7f62fb54afa4354af8409a4bb2358c2699a1003b733e6fce5"
"go1.23.10.linux-amd64.tar.gz" = "sha256:535f9f81802499f2a7dbfa70abb8fda3793725fcc29460f719815f6e10b5fd60"
[tools.golangci-lint]
version = "2.2.2"
backend = "aqua:golangci/golangci-lint"
[tools.golangci-lint.checksums]
"golangci-lint-2.2.2-darwin-arm64.tar.gz" = "sha256:d84d94d042c0d495fd1746f3d18948a75de163b17a14e8de3ef840928dd2df74"
"golangci-lint-2.2.2-linux-amd64.tar.gz" = "sha256:c27fbde948a87d326feacd21df2f61a9c54dbd2e3bfa185c0a1cd6917a6f964f"
[tools.protoc]
version = "27.3"
backend = "aqua:protocolbuffers/protobuf/protoc"
-1
View File
@@ -3,7 +3,6 @@ experimental = true
[tools]
go = "1.23"
golangci-lint = "2.2.2"
protoc = "27.3"
protoc-gen-go = "1.34.2"
protoc-gen-go-grpc = "1.5.1"
-244
View File
@@ -1,244 +0,0 @@
# AI.md - Uncloud Project Guide
This document provides comprehensive information about the Uncloud project for AI assistants to understand the codebase, architecture, and development practices.
## Project Overview
**Uncloud** is a lightweight clustering and container orchestration tool that enables deployment and management of web applications across cloud VMs and bare metal servers. It creates a secure WireGuard mesh network between Docker hosts and provides automatic service discovery, load balancing, HTTPS ingress, and simple CLI commands for application management.
### Key Characteristics
- **Language**: Go
- **Architecture**: Decentralized, no control plane
- **Target**: Self-hosted infrastructure without Kubernetes complexity
- **License**: View LICENSE file for details
- **Status**: Active development, not yet ready for production
## Core Features
### 🏗️ Infrastructure
- **Multi-machine deployment**: Combine cloud VMs, dedicated servers, and bare metal
- **Zero-config networking**: Automatic WireGuard mesh with NAT traversal
- **Decentralized design**: No central control plane, all machines are equal
- **Service discovery**: Built-in DNS server resolves service names to container IPs
### 🚀 Application Management
- **Docker Compose compatibility**: Uses familiar Docker Compose format
- **Zero-downtime deployments**: Rolling updates without service interruption
- **Automatic HTTPS**: Caddy reverse proxy with Let's Encrypt integration
- **Managed DNS**: Free `*.cluster.uncloud.run` subdomains via Uncloud DNS service
- **Cross-machine scaling**: Run containers across multiple machines
### 🔧 Developer Experience
- **Docker-like CLI**: Familiar commands (`uc` binary)
- **Imperative operations**: Direct commands vs. declarative state reconciliation
- **Remote management**: Control entire infrastructure via SSH to any machine
- **Minimal overhead**: ~150MB RAM footprint per machine
## Architecture
### Core Components
1. **CLI (`uc`)** - Main user interface for cluster management
2. **Daemon (`uncloudd`)** - Machine daemon running on each node
3. **Corrosion** - Distributed SQLite database for cluster state (Fly.io project)
4. **Caddy** - Reverse proxy for HTTPS termination and routing
5. **WireGuard** - Secure mesh networking between machines
### Network Architecture
- Each machine gets unique subnet (e.g., `10.210.0.0/24`, `10.210.1.0/24`)
- Containers get cluster-unique IPs for direct communication
- Automatic peer discovery and key management
- NAT traversal for machines behind firewalls
### State Management
- **CRDT-based distributed storage** using Corrosion
- **Eventually consistent** state across all machines
- **Gossip protocol** (Serf) for state propagation
- **No quorum requirements** - partial network splits remain functional
## Project Structure
### Key Directories
- **`cmd/`**: Contains main applications
- `uncloud/`: CLI tool with subcommands for machine, service, volume management
- `uncloudd/`: Daemon that runs on each machine
- `ucind/`: Development cluster management for testing
- **`internal/`**: Internal implementation packages
- `cli/`: Command-line interface logic
- `machine/`: Machine lifecycle and state management
- `daemon/`: Daemon implementation and gRPC services
- `dns/`: Internal DNS server for service discovery
- **`pkg/`**: Public API packages for external use
- `api/`: Core API types and definitions
- `client/`: Client libraries for interacting with Uncloud
- **`experiment/`**: Experimental features and prototypes
- **`scripts/`**: Installation and utility scripts
- **`test/`**: Test suites and test infrastructure
- **`website/`**: Documentation website (Docusaurus)
- **`misc/`**: Design documents and guides
## Key Technologies
### Core Dependencies
```go
// Networking and orchestration
github.com/docker/docker // Docker API client
github.com/docker/compose/v2 // Docker Compose integration
golang.zx2c4.com/wireguard // WireGuard implementation
github.com/hashicorp/serf // Gossip protocol
// State management
github.com/ipfs/go-ds-crdt // CRDT distributed storage
github.com/dgraph-io/badger/v3 // Embedded database
// Web proxy
github.com/caddyserver/caddy/v2 // HTTP server and reverse proxy
// CLI and UX
github.com/spf13/cobra // CLI framework
github.com/charmbracelet/huh // Interactive forms
// gRPC and networking
google.golang.org/grpc // gRPC framework
github.com/siderolabs/grpc-proxy // gRPC proxy for forwarding
```
## Development Workflow
### Build and Development
```bash
# Build binaries
go build -o uncloud ./cmd/uncloud
go build -o uncloudd ./cmd/uncloudd
```
### Key Make Targets
- `proto`: Generate protobuf code
- `ucind-cluster`: Create development cluster
- `update-dev`: Deploy to development machines
- `demo-reset`: Reset demo environment
- `fmt`: Format code
- `test`: Run all tests
- `lint`: Lint the code using golangci-lint
- `lint-and-fix`: Lint the code and fix issues whenever possible
## CLI Commands Structure
The `uc` CLI provides these main command groups:
### Machine Management
```bash
uc machine init <user@host> # Initialize new cluster
uc machine add <user@host> # Add machine to cluster
uc machine ls # List machines
uc machine rm <name> # Remove machine
```
### Service Management
```bash
uc run <image> # Run container from image
uc deploy # Deploy from compose.yaml
uc scale <service> <count> # Scale service replicas
uc ls # List services
uc rm <service> # Remove service
```
### Context and Connectivity
```bash
uc context ls # List available contexts
uc context use <name> # Switch context
```
### Global Flags
- `--connect`: Connect to remote machine directly, without a config file
- `--uncloud-config`: Override config file path
## Development Guidelines
### Code Organization
- **Package naming**: Use clear, descriptive names
- **Error handling**: Wrap errors with context using `fmt.Errorf`
- **Logging**: Use structured logging with levels
- **gRPC**: Services defined in `internal/machine/api/pb/`
### Testing
- Unit tests alongside source files (`*_test.go`)
- Integration tests in `test/e2e/`
- Test fixtures in `test/fixtures/`
### Dependencies
- Prefer standard library when possible
- Pin versions in `go.mod`
- Document rationale for external dependencies
### Configuration
- Support environment variables for key settings
- Validate configuration early
- Provide sensible defaults
## Troubleshooting and Debugging
### Common Issues
- **Networking**: Check WireGuard status, iptables rules
- **DNS**: Verify service discovery resolution
- **Containers**: Use standard Docker debugging tools
- **State sync**: Check Corrosion logs for replication issues
### Debugging Tools
- Standard Linux networking tools (`ping`, `traceroute`, `wireshark`)
- Docker commands (`docker ps`, `docker logs`)
- SSH access to machines for direct inspection
- gRPC debugging tools
### Logs and Monitoring
- Systemd services (getting logs via `journalctl -u SERVICE_NAME`)
- `uncloud` -- Uncloud daemon
- `uncloud-corrosion` -- Corrosion process
- Machine daemon logs
- Container logs via Docker
## File Patterns and Conventions
### Important Files to Understand
- `cmd/uncloud/main.go`: CLI entry point and command structure
- `internal/cli/cli.go`: CLI implementation and configuration
- `internal/machine/machine.go`: Core machine management
- `pkg/api/`: Public API definitions
- `misc/design.md`: Architecture and design philosophy
- `README.md`: User-facing documentation
### Configuration Files
- `go.mod/go.sum`: Go dependency management
- `Makefile`: Build and development tasks
- `Dockerfile`: Container build instructions forUncloud-in-Docker (used for testing)
This document should help AI assistants understand the project structure, make informed suggestions, and contribute effectively to the Uncloud codebase.
-1
View File
@@ -1 +0,0 @@
./AI.md
+5 -24
View File
@@ -1,5 +1,6 @@
CORROSION_IMAGE ?= ghcr.io/psviderski/corrosion:latest
UCIND_IMAGE ?= ghcr.io/psviderski/ucind:latest
DOCS_IMAGE ?= ghcr.io/psviderski/uncloud-docs:latest
update-dev:
GOOS=linux GOARCH=amd64 go build -o uncloudd-linux-amd64 ./cmd/uncloudd && \
@@ -45,11 +46,6 @@ proto:
protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative \
--proto_path=. --proto_path=internal/machine/api/vendor internal/machine/api/pb/*.proto
.PHONY: proto-mise
proto-mise:
mise exec -- protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative \
--proto_path=. --proto_path=internal/machine/api/vendor internal/machine/api/pb/*.proto
.PHONY: corrosion-image
corrosion-image:
docker build -t "$(CORROSION_IMAGE)" --target corrosion .
@@ -71,13 +67,9 @@ test:
ifeq ($(TEST_NAME),)
go test -count=1 -v ./...
else
go test -race -count=1 -v -run ^$(TEST_NAME)$$ ./...
go test -count=1 -v -run ^$(TEST_NAME)$$ ./...
endif
.PHONY: test-e2e
test-e2e:
go test -race -count=1 -v ./test/e2e
.PHONY: test-clean
test-clean:
@CONTAINERS=$$(docker ps --filter "name=ucind-test" -q); \
@@ -97,17 +89,6 @@ test-clean:
vet:
go vet ./...
.PHONY: format fmt
format fmt:
GOOS=linux golangci-lint fmt
LINT_TARGETS := lint lint-and-fix
.PHONY: $(LINT_TARGETS) _lint
$(LINT_TARGETS): _lint
lint: ARGS=
lint-and-fix: ARGS=--fix
_lint:
# Explicitly set OS to Linux to not skip *_linux.go files when running on macOS.
# Uncloud daemon won't likely support OS other than Linux anytime soon, so for now we can rely on that.
GOOS=linux golangci-lint run $(ARGS)
.PHONY: docs-image-push
docs-image:
docker buildx build --push --platform linux/amd64,linux/arm64 -t "$(DOCS_IMAGE)" ./docs
+10 -19
View File
@@ -1,10 +1,10 @@
<div align="center">
<img src="./website/landing/images/logo.svg" height="100" width="100" alt="Uncloud logo"/>
<img src="./website/images/logo.svg" height="100" width="100" alt="Uncloud logo"/>
<h1>Uncloud</h1>
<p><strong>Docker simplicity. Multi-machine power.</strong></p>
<p>
<a href="https://uncloud.run/docs"><img src="https://img.shields.io/badge/Docs-blue.svg?style=for-the-badge&logo=gitbook&logoColor=white" alt="Documentation"></a>
<a href="https://docs.uncloud.run"><img src="https://img.shields.io/badge/Docs-blue.svg?style=for-the-badge&logo=gitbook&logoColor=white" alt="Documentation"></a>
<a href="https://discord.gg/eR35KQJhPu"><img src="https://img.shields.io/badge/discord-5865F2.svg?style=for-the-badge&logo=discord&logoColor=white" alt="Join Discord"></a>
<a href="https://x.com/psviderski"><img src="https://img.shields.io/badge/follow-black?style=for-the-badge&logo=X&logoColor=while" alt="Follow on X"></a>
<a href="https://github.com/sponsors/psviderski"><img src="https://img.shields.io/badge/Donate-EA4AAA.svg?style=for-the-badge&logo=githubsponsors&logoColor=white" alt="Donate"></a>
@@ -52,11 +52,11 @@ complexity of Kubernetes.
## 🎬 Quick demo
The screenshot below demonstrates how I use Uncloud to deploy https://uncloud.run website to 2 remote machines from
the [`compose.yaml`](website/compose.yaml) file on my local machine.
The screenshot below demonstrates how I use Uncloud to deploy the [Uncloud Documentation](https://docs.uncloud.run)
website to 2 remote machines (why not?) from the [`compose.yaml`](docs/compose.yaml) file on my local machine.
It exposes the container port `8000/tcp` as HTTPS on the domain `uncloud.run`, served by the Caddy reverse proxy on the
remote machines. All managed by Uncloud.
It exposes the container port `8000/tcp` as HTTPS on the domain `docs.uncloud.run`, served by the Caddy reverse proxy on
the remote machines. All managed by Uncloud.
![Uncloud compose deployment demo](.github/images/compose-deploy.jpg)
@@ -97,7 +97,7 @@ platform, whether you're running on a $5 VPS, a spare Mac mini, or a rack of bar
curl -fsS https://get.uncloud.run/install.sh | sh
```
See [Installation](https://uncloud.run/docs/getting-started/install-cli) for more options.
See [Installation](https://docs.uncloud.run/getting-started/install-cli) for more options.
2. Initialise your first machine:
@@ -130,15 +130,15 @@ platform, whether you're running on a $5 VPS, a spare Mac mini, or a rack of bar
uncloud-uninstall
```
View the [Documentation](https://uncloud.run/docs) for more information.
View the [Documentation](https://docs.uncloud.run) for more information.
## ⚙️ How it works
Check out the [design document](misc/design.md) to understand Uncloud's design philosophy and goals.
Check out the [design document](docs/design.md) to understand Uncloud's design philosophy and goals.
Here is a diagram of an Uncloud multi-provider cluster of 3 machines:
![Diagram: multi-provider cluster of 3 machines](website/landing/images/diagram.webp)
![Diagram: multi-provider cluster of 3 machines](website/images/diagram.webp)
<details>
<summary>Peek under the hood to see what happens when you run certain commands.</summary>
@@ -323,15 +323,6 @@ SQLite database used to share Uncloud's cluster state.
features, and be the first to know when it's ready for production use.
* Watch this repository for releases.
## 💖 Sponsors
These companies and projects are helping Uncloud with their generous sponsorship and/or services:
<!-- Sentry -->
<a href="https://sentry.io/welcome/">
<img height="100" alt="Sentry" src="https://github.com/user-attachments/assets/6c1439c0-d20d-40dc-a669-c9aa94651dfa" />
</a>
## ❤️ Contributors
Thank you [@cedws](https://github.com/cedws) for being the first contributor to Uncloud! 🎉
+1 -2
View File
@@ -2,9 +2,8 @@ package cluster
import (
"fmt"
"github.com/psviderski/uncloud/internal/ucind"
"github.com/spf13/cobra"
"github.com/psviderski/uncloud/internal/ucind"
)
func NewCreateCommand() *cobra.Command {
+1 -2
View File
@@ -2,9 +2,8 @@ package cluster
import (
"fmt"
"github.com/psviderski/uncloud/internal/ucind"
"github.com/spf13/cobra"
"github.com/psviderski/uncloud/internal/ucind"
)
func NewRemoveCommand() *cobra.Command {
+1 -8
View File
@@ -21,7 +21,6 @@ type deployOptions struct {
profiles []string
services []string
noBuild bool
recreate bool
context string
}
@@ -51,8 +50,6 @@ func NewDeployCommand() *cobra.Command {
"Name of the cluster context to deploy to (default is the current context)")
cmd.Flags().BoolVarP(&opts.noBuild, "no-build", "n", false,
"Do not build images before deploying services. (default false)")
cmd.Flags().BoolVar(&opts.recreate, "recreate", false,
"Recreate containers even if their configuration and image haven't changed.")
// TODO: Consider adding a filter flag to specify which machines to deploy to but keep the rest running.
// Could be useful to test a new version on a subset of machines before rolling out to all.
@@ -111,11 +108,7 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
}
defer clusterClient.Close()
var strategy deploy.Strategy
if opts.recreate {
strategy = &deploy.RollingStrategy{ForceRecreate: true}
}
composeDeploy, err := compose.NewDeploymentWithStrategy(ctx, clusterClient, project, strategy)
composeDeploy, err := compose.NewDeployment(ctx, clusterClient, project)
if err != nil {
return fmt.Errorf("create compose deployment: %w", err)
}
+6 -7
View File
@@ -41,7 +41,7 @@ func NewAddCommand() *cobra.Command {
if err != nil {
return fmt.Errorf("parse remote machine: %w", err)
}
remoteMachine := &cli.RemoteMachine{
remoteMachine := cli.RemoteMachine{
User: user,
Host: host,
Port: port,
@@ -59,12 +59,11 @@ func NewAddCommand() *cobra.Command {
cmd.Flags().StringVar(
&opts.publicIP, "public-ip", "auto",
"Public IP address of the machine for ingress configuration. Use 'auto' for automatic detection, "+
fmt.Sprintf("blank '' or '%s' to disable ingress on this machine, or specify an IP address.", PublicIPNone),
"blank '' or 'none' to disable ingress on this machine, or specify an IP address.",
)
cmd.Flags().StringVarP(
&opts.sshKey, "ssh-key", "i", "",
fmt.Sprintf("Path to SSH private key for remote login (if not already added to SSH agent). (default %q)",
cli.DefaultSSHKeyPath),
&opts.sshKey, "ssh-key", "i", "~/.ssh/id_ed25519",
"Path to SSH private key for remote login (if not already added to SSH agent).",
)
cmd.Flags().StringVar(
&opts.version, "version", "latest",
@@ -78,12 +77,12 @@ func NewAddCommand() *cobra.Command {
return cmd
}
func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine, opts addOptions) error {
func add(ctx context.Context, uncli *cli.CLI, remoteMachine cli.RemoteMachine, opts addOptions) error {
var publicIP *netip.Addr
switch opts.publicIP {
case "auto":
publicIP = &netip.Addr{}
case "", PublicIPNone:
case "", "none":
publicIP = nil
default:
ip, err := netip.ParseAddr(opts.publicIP)
-6
View File
@@ -1,6 +0,0 @@
package machine
const (
// PublicIPNone is the value used to indicate removal of public IP
PublicIPNone = "none"
)
+4 -5
View File
@@ -77,12 +77,11 @@ func NewInitCommand() *cobra.Command {
cmd.Flags().StringVar(
&opts.publicIP, "public-ip", "auto",
"Public IP address of the machine for ingress configuration. Use 'auto' for automatic detection, "+
fmt.Sprintf("blank '' or '%s' to disable ingress on this machine, or specify an IP address.", PublicIPNone),
"blank '' or 'none' to disable ingress on this machine, or specify an IP address.",
)
cmd.Flags().StringVarP(
&opts.sshKey, "ssh-key", "i", "",
fmt.Sprintf("Path to SSH private key for remote login (if not already added to SSH agent). (default %q)",
cli.DefaultSSHKeyPath),
&opts.sshKey, "ssh-key", "i", "~/.ssh/id_ed25519",
"Path to SSH private key for remote login (if not already added to SSH agent).",
)
cmd.Flags().StringVar(
&opts.version, "version", "latest",
@@ -106,7 +105,7 @@ func initCluster(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteM
switch opts.publicIP {
case "auto":
publicIP = &netip.Addr{}
case "", PublicIPNone:
case "", "none":
publicIP = nil
default:
ip, err := netip.ParseAddr(opts.publicIP)
-47
View File
@@ -1,47 +0,0 @@
package machine
import (
"context"
"fmt"
"github.com/psviderski/uncloud/internal/cli"
"github.com/spf13/cobra"
)
func NewRenameCommand() *cobra.Command {
var contextName string
cmd := &cobra.Command{
Use: "rename OLD_NAME NEW_NAME",
Short: "Rename a machine in the cluster.",
Long: `Rename a machine in the cluster.
This command changes the name of an existing machine while preserving all other
configuration including network settings, public IP, and cluster membership.`,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
uncli := cmd.Context().Value("cli").(*cli.CLI)
return rename(cmd.Context(), uncli, contextName, args[0], args[1])
},
}
cmd.Flags().StringVarP(
&contextName, "context", "c", "",
"Name of the cluster context. (default is the current context)",
)
return cmd
}
func rename(ctx context.Context, uncli *cli.CLI, contextName, oldName, newName string) error {
client, err := uncli.ConnectCluster(ctx, contextName)
if err != nil {
return err
}
defer client.Close()
machine, err := client.RenameMachine(ctx, oldName, newName)
if err != nil {
return fmt.Errorf("rename machine: %w", err)
}
fmt.Printf("Machine %q renamed to %q (ID: %s)\n", oldName, machine.Name, machine.Id)
return nil
}
+21 -70
View File
@@ -14,13 +14,12 @@ import (
"github.com/docker/compose/v2/pkg/progress"
"github.com/docker/docker/api/types/container"
"github.com/psviderski/uncloud/internal/cli"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/pkg/api"
"github.com/spf13/cobra"
)
type removeOptions struct {
noReset bool
force bool
yes bool
context string
}
@@ -31,7 +30,7 @@ func NewRmCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "rm MACHINE",
Aliases: []string{"remove", "delete"},
Short: "Remove a machine from a cluster and reset it.",
Short: "Remove a machine from a cluster.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
uncli := cmd.Context().Value("cli").(*cli.CLI)
@@ -43,14 +42,11 @@ func NewRmCommand() *cobra.Command {
"Name of the cluster context. (default is the current context)")
cmd.Flags().BoolVarP(&opts.yes, "yes", "y", false,
"Do not prompt for confirmation before removing the machine.")
cmd.Flags().BoolVar(&opts.noReset, "no-reset", false,
"Do not reset the machine after removing it from the cluster. This will leave all containers and data intact.")
return cmd
}
func remove(ctx context.Context, uncli *cli.CLI, nameOrID string, opts removeOptions) error {
// TODO: automatically choose a connection to the machine that is not being removed.
func remove(ctx context.Context, uncli *cli.CLI, machineName string, opts removeOptions) error {
client, err := uncli.ConnectCluster(ctx, opts.context)
if err != nil {
return fmt.Errorf("connect to cluster: %w", err)
@@ -58,46 +54,22 @@ func remove(ctx context.Context, uncli *cli.CLI, nameOrID string, opts removeOpt
defer client.Close()
// Verify the machine exists and list all service containers on it including stopped ones.
mctx, machines, err := api.ProxyMachinesContext(ctx, client, []string{nameOrID})
listCtx, machines, err := api.ProxyMachinesContext(ctx, client, []string{machineName})
if err != nil {
return err
}
if len(machines) == 0 {
return fmt.Errorf("machine '%s' not found in the cluster", nameOrID)
return fmt.Errorf("machine '%s' not found in the cluster", machineName)
}
m := machines[0].Machine
// Verify if the machine being removed is the proxy machine we're connected to.
proxyMachine, err := client.MachineClient.Inspect(ctx, nil)
if err != nil {
return fmt.Errorf("inspect proxy machine: %w", err)
}
if proxyMachine.Id == m.Id {
allMachines, err := client.ListMachines(ctx, nil)
if err != nil {
return fmt.Errorf("list machines: %w", err)
}
if len(allMachines) > 1 {
return errors.New("cannot remove the machine you are currently connected to. " +
"Please connect to another machine in the cluster and try again. " +
"Use --connect flag or update 'connections' for the cluster context in your Uncloud config")
// It's ok to remove the proxy machine if it's the last one in the cluster.
}
}
// TODO: mark the machine as being removed and unschedulable when this is possible to prevent new containers
// from being scheduled on it while the removal is in progress.
reset := !opts.noReset
var containers []api.ServiceContainer
reachable := false
if reset {
// Check if the machine is up and has service containers.
listOpts := container.ListOptions{All: true}
machineContainers, err := client.Docker.ListServiceContainers(mctx, "", listOpts)
if err == nil {
reachable = true
containers = machineContainers[0].Containers
machineContainers, err := client.Docker.ListServiceContainers(listCtx, "", listOpts)
if err != nil {
return fmt.Errorf("list containers: %w", err)
}
containers := machineContainers[0].Containers
if len(containers) > 0 {
plural := ""
if len(containers) > 1 {
@@ -106,18 +78,11 @@ func remove(ctx context.Context, uncli *cli.CLI, nameOrID string, opts removeOpt
fmt.Printf("Found %d service container%s on machine '%s':\n", len(containers), plural, m.Name)
fmt.Println(formatContainerTree(containers))
fmt.Println()
fmt.Println("This will remove all service containers from the machine, remove it from the cluster, " +
"and reset it to the uninitialised state.")
fmt.Println("This will remove all service containers on the machine, reset it to the uninitialised state, " +
"and remove it from the cluster.")
} else {
fmt.Printf("No service containers found on machine '%s'.\n", m.Name)
fmt.Println("This will remove the machine from the cluster and reset it to the uninitialised state.")
}
} else {
fmt.Printf("This will remove machine '%s' from the cluster without resetting it as it's unreachable.\n",
m.Name)
}
} else {
fmt.Printf("This will remove machine '%s' from the cluster without resetting it.\n", m.Name)
fmt.Println("This will reset the machine to the uninitialised state and remove it from the cluster.")
}
if !opts.yes {
@@ -131,37 +96,23 @@ func remove(ctx context.Context, uncli *cli.CLI, nameOrID string, opts removeOpt
}
}
if reset && len(containers) > 0 {
if len(containers) > 0 {
err = progress.RunWithTitle(ctx, func(ctx context.Context) error {
return removeContainers(ctx, client, containers)
}, uncli.ProgressOut(), "Removing containers")
if err != nil {
return fmt.Errorf("remove containers: %w", err)
}
fmt.Println()
}
if _, err = client.RemoveMachine(ctx, &pb.RemoveMachineRequest{Id: m.Id}); err != nil {
return fmt.Errorf("remove machine from cluster: %w", err)
}
fmt.Printf("Machine '%s' removed from the cluster.\n", m.Name)
// TODO: 4. Implement and call Reset via Machine API to reset the machine state to uninitialised.
// TODO: 5. Remove the machine from the cluster store.
if reset && reachable {
_, err = client.MachineClient.Reset(mctx, &pb.ResetRequest{})
if err != nil {
fmt.Printf("WARNING: Failed to reset machine: %v\n", err)
} else {
fmt.Println("Machine reset initiated and will complete in the background.")
}
}
// TODO: remove the connection to the machine from the uncloud config if it exists. We need a way to associate
// the machine with its connection in the config, e.g. by storing the machine name in the connection metadata.
// TODO: If Caddy was running on this machine and a cluster domain is reserved,
// let the user know that the DNS records should be updated.
return nil
return fmt.Errorf("resetting machine is not fully implemented yet")
//fmt.Printf("Machine '%s' removed from the cluster.\n", m.Name)
//return nil
}
// formatContainerTree formats a list of containers grouped by service as a tree structure.
-2
View File
@@ -14,9 +14,7 @@ func NewRootCommand() *cobra.Command {
NewAddCommand(),
NewInitCommand(),
NewListCommand(),
NewRenameCommand(),
NewRmCommand(),
NewUpdateCommand(),
NewTokenCommand(),
)
return cmd
+1 -2
View File
@@ -2,10 +2,9 @@ package machine
import (
"fmt"
"github.com/spf13/cobra"
"github.com/psviderski/uncloud/internal/daemon"
"github.com/psviderski/uncloud/internal/machine"
"github.com/spf13/cobra"
)
type tokenOptions struct {
-128
View File
@@ -1,128 +0,0 @@
package machine
import (
"context"
"fmt"
"net/netip"
"github.com/psviderski/uncloud/internal/cli"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/spf13/cobra"
)
type updateOptions struct {
name string
publicIP string
context string
}
func NewUpdateCommand() *cobra.Command {
opts := updateOptions{}
cmd := &cobra.Command{
Use: "update",
Short: "Update machine configuration in the cluster.",
Long: `Update machine configuration in the cluster.
This command allows setting various machine properties including:
- Machine name (--name)
- Public IP address (--public-ip)
At least one flag must be specified to perform an update operation.`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
uncli := cmd.Context().Value("cli").(*cli.CLI)
return update(cmd.Context(), uncli, cmd, opts, args[0])
},
}
cmd.Flags().StringVar(
&opts.name, "name", "",
"New name for the machine",
)
cmd.Flags().StringVar(
&opts.publicIP, "public-ip", "",
fmt.Sprintf("Public IP address of the machine for ingress configuration. Use '%s' or '' to remove the public IP.", PublicIPNone),
)
cmd.Flags().StringVarP(
&opts.context, "context", "c", "",
"Name of the cluster context. (default is the current context)",
)
return cmd
}
func update(ctx context.Context, uncli *cli.CLI, cmd *cobra.Command, opts updateOptions, machineNameOrID string) error {
// Check if at least one flag was explicitly set
if !cmd.Flags().Changed("name") && !cmd.Flags().Changed("public-ip") {
return fmt.Errorf("at least one update flag must be specified (--name, --public-ip)")
}
client, err := uncli.ConnectCluster(ctx, opts.context)
if err != nil {
return err
}
defer client.Close()
// First, resolve the machine to get its ID
machine, err := client.InspectMachine(ctx, machineNameOrID)
if err != nil {
return fmt.Errorf("find machine: %w", err)
}
// Build the update request
req := &pb.UpdateMachineRequest{
MachineId: machine.Machine.Id,
}
if opts.name != "" {
req.Name = &opts.name
}
// Check if --public-ip flag was explicitly provided
if cmd.Flags().Changed("public-ip") {
if opts.publicIP == "" || opts.publicIP == PublicIPNone {
req.PublicIp = &pb.IP{} // Empty IP to signal removal
} else {
// Parse and validate the public IP
ip, err := netip.ParseAddr(opts.publicIP)
if err != nil {
return fmt.Errorf("invalid public IP address %q: %w", opts.publicIP, err)
}
req.PublicIp = pb.NewIP(ip)
}
}
// Perform the update operation
updatedMachine, err := client.UpdateMachine(ctx, req)
if err != nil {
return fmt.Errorf("update machine: %w", err)
}
// Report what was changed
changes := make([]string, 0)
if opts.name != "" {
changes = append(changes, fmt.Sprintf("name: %q -> %q", machine.Machine.Name, updatedMachine.Name))
}
if cmd.Flags().Changed("public-ip") {
oldIP := PublicIPNone
if machine.Machine.PublicIp != nil {
if addr, err := machine.Machine.PublicIp.ToAddr(); err == nil {
oldIP = addr.String()
}
}
newIP := PublicIPNone
if updatedMachine.PublicIp != nil {
if addr, err := updatedMachine.PublicIp.ToAddr(); err == nil {
newIP = addr.String()
}
}
changes = append(changes, fmt.Sprintf("public IP: %s -> %s", oldIP, newIP))
}
fmt.Printf("Machine %q (ID: %s) configuration updated:\n", updatedMachine.Name, updatedMachine.Id)
for _, change := range changes {
fmt.Printf(" %s\n", change)
}
return nil
}
+1
View File
@@ -9,6 +9,7 @@ import (
"github.com/docker/docker/pkg/stringid"
"github.com/docker/go-units"
"github.com/psviderski/uncloud/internal/cli"
"github.com/spf13/cobra"
)
@@ -2,9 +2,7 @@
*
# Allow files and directories.
!blog/
!docs/
!landing/
!src/
!static/
!pkg/
+1 -1
View File
@@ -5,5 +5,5 @@
:8000 {
root * /usr/share/caddy
file_server
log
try_files {path} /index.html
}
+1 -4
View File
@@ -19,12 +19,9 @@ RUN npm ci
## Build the static site.
RUN npm run build
# Stage 3: Serve static Docusaurus site and landing page with Caddy.
# Stage 3: Serve static site with Caddy.
FROM caddy:2.10.0-alpine AS caddy
## Copy the Caddyfile.
COPY ./Caddyfile /etc/caddy/Caddyfile
## Copy the Docusaurus build output.
COPY --from=prod /opt/docusaurus/build /usr/share/caddy
# Copy the landing page assets.
COPY landing/images /usr/share/caddy/images
COPY landing/index.html landing/style.css /usr/share/caddy/
+41
View File
@@ -0,0 +1,41 @@
# Website
This website is built using [Docusaurus](https://docusaurus.io/), a modern static website generator.
### Installation
```
$ yarn
```
### Local Development
```
$ yarn start
```
This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server.
### Build
```
$ yarn build
```
This command generates static content into the `build` directory and can be served using any static contents hosting service.
### Deployment
Using SSH:
```
$ USE_SSH=true yarn deploy
```
Not using SSH:
```
$ GIT_USER=<Your GitHub username> yarn deploy
```
If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch.
@@ -1,31 +1,39 @@
---
title: How to connect Docker containers across multiple hosts with WireGuard
description: Learn how to configure a WireGuard overlay network that lets Docker containers securely communicate
across multiple hosts. No exposed ports needed.
slug: connect-docker-containers-across-hosts-wireguard
image: ./wireguard-overlay.png
authors: psviderski
tags: [ docker, wireguard, networking, vpn ]
---
# WireGuard overlay network for Docker containers
You want your Docker containers to talk to each other, but they're running on different machines. Perhaps across
different cloud providers or mixing cloud with on-prem. The usual approach of mapping services to host ports quickly
becomes a pain. Worse, if they're on the public internet, you need to secure every exposed endpoint with TLS and auth.
# How to connect Docker containers across multiple hosts using WireGuard
What if your containers on different machines could communicate directly without exposing any ports? Using their private
Docker IPs, as if they were on the same machine. Here's how you can use pure WireGuard and some clever networking tricks
to make this work.
# Connect Docker containers across multiple hosts with WireGuard
You have Docker containers running on different Linux machines. You want container A on one machine to talk directly to
container B on another machine using their private IPs. For example, to run your application and database containers on
separate machines without exposing them publicly. Here's how you can use pure WireGuard and some networking tricks to
make this work.
I implemented this technique to enable cross-machine container communication in
[Uncloud](https://github.com/psviderski/uncloud), an open source clustering and deployment tool for Docker.
* [What we're building](#what-were-building)
* [Prequisites](#prerequisites)
* [Step 1: Configure Docker networks](#step-1-configure-docker-networks)
* [Step 2: Connect Docker networks with WireGuard](#step-2-connect-docker-networks-with-wireguard)
* [Step 3: Configure IP routing](#step-3-configure-ip-routing)
* [Step 4: Testing](#step-4-testing)
* [Step 5: Make the configuration persistent](#step-5-make-the-configuration-persistent)
* [Scaling beyond two machines and limitations](#scaling-beyond-two-machines-and-limitations)
* [Automating with Uncloud](#automating-with-uncloud)
* [Alternative solutions](#alternative-solutions)
* [Conclusion](#conclusion)
## What we're building
Docker containers are typically connected to a [bridge network](https://docs.docker.com/engine/network/drivers/bridge/)
on their host machine, which allows them to communicate with each other. A bridge network also provides isolation from
containers not connected to it and other networks on the host. What we want to achieve is connecting these bridge
containers not connected to it and other networks on the host. What we want to achieve is to connect these bridge
networks across machines so that containers on different machines can communicate as if they were connected to the same
local bridge network.
The incantation we need is called a site-to-site VPN. Any solution would work. Moreover, if the machines are on the same
local network, they're already connected and only lack the appropriate routing configuration. But I'll describe a more
local network, they're already connected and only miss the appropriate routing configuration. But I'll describe a more
versatile approach that works even when the machines are on different continents or behind NAT. WireGuard is the ideal
solution for this use case: it's lightweight, [fast](https://www.wireguard.com/performance/), simple to configure,
provides [strong security](https://www.wireguard.com/protocol/) and NAT traversal.
@@ -37,21 +45,19 @@ communicate with each other using their private IPs.
I will use these two machines:
* Machine 1: Debian 12 virtual machine in my homelab network in Australia, which is behind NAT
* Machine 1: Debian 12 virtual machine in my homelab network in Australia which is behind NAT
* Machine 2: Ubuntu 24.04 server from Hetzner in Finland that has a public IP
![WireGuard overlay network](wireguard-overlay.png)
<!-- truncate -->
![wireguard-overlay.png](wireguard-overlay.png)
## Prerequisites
- Basic knowledge of [Docker networking](https://docs.docker.com/network/) and [WireGuard](https://www.wireguard.com/).
* Basic knowledge of [Docker networking](https://docs.docker.com/network/) and [WireGuard](https://www.wireguard.com/).
If you're new to these topics, you might want to read up on them first.
- At least two Linux machines with root access and Docker installed. They should be on the same network or be able to
communicate over the internet.
* At least two Linux machines with root access and Docker installed. They should be on the same network or reachable
over the internet.
## Step 1: Configure Docker networks
# Step 1: Configure Docker networks
Most of the commands in this guide require root privileges. You can run them with `sudo` or log in as root. I'll start
root shells on both machines with `sudo -i` for convenience.
@@ -64,7 +70,7 @@ Therefore, let's create new Docker bridge networks on each machine with manually
choose any subnets from
the [private IPv4 address ranges](https://en.wikipedia.org/wiki/Private_network#Private_IPv4_addresses)
that do not overlap with each other or with your existing networks. I'll use `10.200.1.0/24` and `10.200.2.0/24`
for Machine 1 and Machine 2, respectively. They don't even need to be sequential or be part of the same larger network.
for Machine 1 and Machine 2 respectively. They don't even need to be sequential or be part of the same larger network.
However, using a common parent network (like `10.200.0.0/16` in my case) can simplify firewall rules and make it easier
to manage more machines later.
@@ -81,16 +87,16 @@ Starting with Docker 28.2.0 ([PR](https://github.com/moby/moby/pull/49832)), you
host interfaces you
allow [direct routing](https://docs.docker.com/engine/network/packet-filtering-firewalls/#direct-routing) to containers
in bridge networks. This is done by specifying the `com.docker.network.bridge.trusted_host_interfaces` option when
creating the network. In our case, we want to allow routing via the WireGuard interface `wg0` that will be created in
the next step.
creating the network. In our case, we want to allow routing via the WireGuard interface `wg0` that we be created in the
next step.
Provide this option even if you're using an older Docker version, as it'll be required if you upgrade Docker in the
Provide this option even if you're using an older Docker version as it'll be required if you upgrade Docker in the
future.
## Step 2: Connect Docker networks with WireGuard
By default, WireGuard uses the UDP port 51820 for communication. To establish a tunnel, at least one of the machines
needs to be able to reach the other's port over the internet or local network. Please make sure it's not blocked by a
By default, WireGuard uses the UDP port 51280 for communication. To establish a tunnel, at least one of the machines
need to be able to reach the other's port over the internet or local network. Please make sure it's not blocked by a
firewall on both machines.
For example, when using `iptables`, you can allow incoming UDP traffic on port 51820 with the following command:
@@ -122,7 +128,7 @@ PrivateKey = <replace with 'privatekey' file content from Machine 1>
[Peer]
PublicKey = <replace with 'publickey' file content from Machine 2>
# IP ranges for which a peer will route traffic: Docker subnet on Machine 2
# IP ranges for which a peer will route traffic - Docker subnet on Machine 2
AllowedIPs = 10.200.2.0/24
# Public IP of Machine 2
Endpoint = 157.180.72.195:51820
@@ -139,7 +145,7 @@ PrivateKey = <replace with 'privatekey' file content from Machine 2>
[Peer]
PublicKey = <replace with 'publickey' file content from Machine 1>
# IP ranges for which a peer will route traffic: Docker subnet on Machine 1
# IP ranges for which a peer will route traffic - Docker subnet on Machine 1
AllowedIPs = 10.200.1.0/24
# Reachable endpoint of Machine 1
# Endpoint =
@@ -156,14 +162,14 @@ In my case, Machine 1 is behind NAT in my private homelab network which is not r
server (Machine 2). The bidirectional tunnel can still be established in this case but Machine 1 must initiate the
connection.
If both of your machines are reachable from each other, you should specify the `Endpoint` option in both configs which
If both of your machine are reachable from each other, you should specify the `Endpoint` option in both configs which
will allow them to establish the connection without waiting for the other side to initiate it. If both of your machines
are behind NAT, see [NAT to NAT Connections](https://github.com/pirate/wireguard-docs#NAT-to-NAT-Connections) for more
information.
Note also that we don't set the `Address` option in the configs because we don't want to assign any IP addresses to the
Note also that we don't set `Address` option in the configs because we don't want to assign any IP addresses to the
WireGuard interfaces. We want the tunnel to only encapsulate and transfer packets from the `multi-host` bridge networks
and don't want either end of it to be the destination for the packets.
and don't want any end of it to be the destination for the packets.
As the key pairs are now specified in the configuration files, you can remove the `privatekey` and `publickey` files on
both machines:
@@ -199,19 +205,14 @@ If you see the `latest handshake` time updating, it means the tunnel is working
## Step 3: Configure IP routing
You've established the WireGuard tunnel, but packets between containers won't flow yet. You need to configure IP routing
between the tunnel and the container networks.
Docker daemon automatically enables IP forwarding in the kernel when it starts, so you don't need to manually configure
`net.ipv4.ip_forward` with `sysctl`.
The challenge is that Docker blocks traffic between external interfaces and container networks by default for security
reasons. You need to explicitly allow WireGuard traffic from `wg0` interface to reach your containers via the
`multi-host` bridge interface. Docker uses iptables, so you can allow this traffic by adding a rule to the `FORWARD`
chain before any other Docker-managed rules that would drop it.
Fortunately, Docker creates a special `DOCKER-USER` chain exactly for this purpose. It's processed before other
Docker-managed chains, allowing you to add custom rules that won't be overridden by Docker.
However, Docker blocks traffic between external interfaces and container networks by default for security. You need to
explicitly allow WireGuard traffic from `wg0` interface to reach your containers via the `multi-host` bridge interface.
Docker uses iptables, so you can allow this traffic by adding a rule to the `FORWARD` chain before any other
Docker-managed rules that would drop it. Luckily, Docker creates a special `DOCKER-USER` chain exactly for this purpose
that the `FORWARD` chain jumps to before jumping to any other Docker-managed chains.
To create the required iptables rule, you need to find the bridge interface name for the `multi-host` network you
created earlier. It's named `br-<short-network-id>`, where `<short-network-id>` is the first 12 characters of the
@@ -219,7 +220,7 @@ network ID.
Add the iptables rule to allow traffic from `wg0` to `multi-host` bridge on Machine 1:
```shell
```bash
$ docker network ls -f name=multi-host
NETWORK ID NAME DRIVER SCOPE
661096b2a5d9 multi-host bridge local
@@ -228,14 +229,14 @@ $ iptables -I DOCKER-USER -i wg0 -o br-661096b2a5d9 -j ACCEPT
Add the iptables rule to allow traffic from `wg0` to `multi-host` bridge on Machine 2:
```shell
```bash
$ docker network ls -f name=multi-host
NETWORK ID NAME DRIVER SCOPE
48f808048e7c multi-host bridge local
$ iptables -I DOCKER-USER -i wg0 -o br-48f808048e7c -j ACCEPT
```
The traffic in the other direction (from `multi-host` bridge to `wg0`) is not blocked by Docker by default. But it still
The traffic the other way around (from `multi-host` bridge to `wg0`) is not blocked by Docker by default. But it still
won't be able to make it through the tunnel. The reason is that Docker creates a `MASQUERADE` rule in the `nat` table
for every bridge network with option
[`com.docker.network.bridge.enable_ip_masquerade`](https://docs.docker.com/engine/network/drivers/bridge/#options) set
@@ -245,13 +246,13 @@ to `true` (which is the default). In my case, the rule looks like this on Machin
POSTROUTING -s 10.200.1.0/24 ! -o br-661096b2a5d9 -j MASQUERADE
```
This essentially configures NAT for all external traffic coming from containers which is necessary for allowing them to
This essentially configures NAT for all external traffic coming from containers which is necessary to allow them to
access the internet and other external networks. However, it equally applies to the traffic going through the `wg0`
interface. It tries to masquerade the source IP address of the packets with the IP address of the `wg0` interface and
fails because the `wg0` interface doesn't have an IP. This results in the packets being
[dropped](https://elixir.bootlin.com/linux/v6.15.5/source/net/netfilter/nf_nat_masquerade.c#L54-L58).
You could assign an IP address to `wg0` but this would cause the following unwanted side effects:
You cloud assign an IP address to `wg0` but this would cause the following unwanted side effects:
- Containers from other Docker networks on the same machine could route through the tunnel to reach remote `multi-host`
containers, violating Docker's network isolation model.
@@ -321,7 +322,7 @@ PING 10.200.2.2 (10.200.2.2): 56 data bytes
64 bytes from 10.200.2.2: seq=2 ttl=62 time=297.285 ms
```
Both hosts have IPs assigned to the `multi-host` bridges, `10.200.1.1` and `10.200.2.1` respectively, which should also
Both hosts have IPs assigned to the `multi-host` bridges, `10.200.1.1` and `10.200.2.1` respectively which should aslo
be reachable from the containers or hosts on both machines.
You can see from the `ping` command the latency is quite high (~300 ms) in my case because the packets have to travel
@@ -343,12 +344,12 @@ be to use `PostUp` and `PostDown` options in the WireGuard configs to automatica
starts/stops.
Append the following lines to the `[Interface]` section in `/etc/wireguard/wg0.conf`. Make sure to replace
`<network-id>` with your actual Docker network ID from [Step 3](#step-3-configure-ip-routing). The `%i` is replaced by
WireGuard with the interface name (`wg0`).
`<network-id>` with your actual Docker network ID from Step 3. The `%i` is replaced by WireGuard with the interface
name (`wg0`).
On Machine 1:
```ini
```shell
[Interface]
...
PostUp = iptables -I DOCKER-USER -i %i -o br-<network-id> -j ACCEPT; iptables -t nat -I POSTROUTING -s 10.200.1.0/24 -o %i -j RETURN
@@ -357,7 +358,7 @@ PostDown = iptables -D DOCKER-USER -i %i -o br-<network-id> -j ACCEPT; iptables
On Machine 2:
```ini
```shell
[Interface]
...
PostUp = iptables -I DOCKER-USER -i %i -o br-<network-id> -j ACCEPT; iptables -t nat -I POSTROUTING -s 10.200.2.0/24 -o %i -j RETURN
@@ -367,7 +368,7 @@ PostDown = iptables -D DOCKER-USER -i %i -o br-<network-id> -j ACCEPT; iptables
### Start WireGuard on boot
The `wireguard-tools` package provides a convenient systemd service to manage WireGuard interfaces. Since our iptables
rules should have priority over Docker's rules, WireGuard must start after Docker.
rules should have a priority over Docker's rules, WireGuard must start after Docker.
Create a systemd drop-in configuration for this:
@@ -385,135 +386,54 @@ Then enable the WireGuard service to start on boot:
```shell
systemctl enable wg-quick@wg0.service
systemctl daemon-reload
# Verify the unit includes the drop-in configuration
# Verify the unit includes the drop-in configuration.
systemctl cat wg-quick@wg0.service
```
## Scaling beyond two machines
## Scaling beyond two machines and limitations
Adding a third machine means following the same steps as above on it and updating WireGuard configs on *all* existing
machines. Each machine needs a `[Peer]` section for every other machine in the network. With 5 machines, that's 4 peer
entries per config file or 20 peer configurations total that establish a full mesh topology.
![WireGuard full mesh](wireguard-mesh.png)
## Limitations
### DNS resolution
The main limitation of this setup is that containers can't find each other by name across machines. You need to use
their IP addresses directly or implement a service discovery solution like Consul or CoreDNS.
For small deployments, you can assign static IPs to containers and use those IPs in your app configuration. But service
discovery is essential for larger and more dynamic deployments.
### NAT traversal constraints
For WireGuard connections to work, at least one machine in each pair must be publicly reachable. The connection fails if
both machines are behind NAT. While solutions exist (STUN/TURN servers, UDP hole punching), they're beyond the scope of
this guide.
Common scenarios that work:
- ✅ Cloud VPS (public or private IP) ↔ Cloud VPS (public or private IP). Both can use private IPs only if they're in the
same cloud provider's network
- ✅ Homelab (behind NAT) ↔ Cloud VPS (public IP)
- ✅ Homelab (private IP) ↔ Homelab (private IP on the same local network)
- ❌ Homelab (behind NAT) ↔ Friend's homelab (behind NAT) — requires a relay server
//Adding a third machine requires updating configs on all existing machines. This gets tedious fast... //WireGuard mesh
and challenges to manually manage key pairs and distribute configs //Requirements for NAT traversal: at least one
machine in each pair must be reachable by the other. The wireguard will fail to establish a connection if both machines
are behind NAT without special tricks that are beyond the scope of this post. DNS resolution for container names across
machines is not covered here, but you can use a service discovery tool like Consul.
## Automating with Uncloud
As your setup grows, managing subnet allocation for Docker networks (ensuring each gets a unique range like
`10.200.1.0/24`, `10.200.2.0/24`) and updating WireGuard configs manually may become tedious quickly.
//I built Uncloud to handle all the heavy lifting automatically.
I built [Uncloud](https://github.com/psviderski/uncloud), an open source clustering and deployment tool for Docker, to
handle all the heavy lifting automatically. You can get the same result and much more with just a few commands.
Initialise a new cluster on your first machine:
You can initialise a cluster of machines by running the following commands:
```shell
uc machine init user@machine1
uc machine init user@machine1-ip
uc machine add user@machine2-ip
...
uc machine add user@machineN-ip
```
Add more machines to the cluster:
//This will create `uncloud` Docker bridge network on each machine with `10.210.N.0/24` subnet by default and set up
//WireGuard mesh network between them and make persistent across reboots.
```shell
uc machine add user@machine2
uc machine add user@machine3
```
This is what these commands do:
- Create the `uncloud` Docker network on each machine with unique subnets (`10.210.0.0/24`, `10.210.1.0/24`, etc.).
- Generate WireGuard key pairs and distribute public keys across machines.
- Start a full mesh WireGuard network.
- Configure iptables rules for container communication.
- Make everything persistent across reboots.
Beyond the network setup, you also get:
- Multi-machine [Docker Compose](https://docs.docker.com/reference/compose-file/) deployments with zero downtime.
- Built-in DNS server that resolves container IPs by their service names.
- Automatic HTTPS and reverse proxy configuration.
Check out the [documentation](https://uncloud.run/docs) for more information.
//Mention embedded DNS that resolves container IPs by their service names and multi-machine Docker Compose support.
## Alternative solutions
Before settling on the WireGuard approach, I evaluated several alternatives. Note that I only considered lightweight
solutions suitable for Docker. Kubernetes and its CNI ecosystem deserve a separate discussion.
//I wanted to explore only lightweight solutions for Docker so not talking about Kubernetes and a numerous CNI
//drivers. Let's leave this beast for another time.
### Docker Swarm overlay network
Docker Swarm includes built-in [overlay networking](https://docs.docker.com/engine/network/drivers/overlay/). However,
to use an overlay network, you need to run a [Swarm cluster](https://docs.docker.com/engine/swarm/) on all machines.
This introduces additional complexity:
- Cluster nodes must
[maintain the quorum](https://docs.docker.com/engine/swarm/admin_guide/#maintain-the-quorum-of-managers). Losing
quorum impacts the functionality of overlay networks.
- Ports 2377, 7946, and 4789 must be exposed to untrusted networks (if connecting machines over the internet) for
cluster management, node communication, and VXLAN overlay traffic.
- VXLAN traffic is unencrypted by default, requiring additional
[hardening](https://docs.docker.com/engine/swarm/swarm-tutorial/#open-protocols-and-ports-between-the-hosts) with
IPSec and firewalls.
- Every node must be publicly reachable. VXLAN fails if machines are behind NAT.
If these limitations are acceptable, an overlay network is a great option. Note that you can use an overlay network with
regular containers without using any other Swarm features.
### Flannel
[Flannel](https://github.com/flannel-io/flannel) is battle-tested in Kubernetes but can also be used with Docker. It
supports multiple [backends](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md) including
VXLAN and WireGuard.
The main caveat is that Flannel requires running etcd as the datastore for coordination. Depending on your availability
requirements, you may need to set up an etcd cluster with multiple nodes. This is not a problem if you're already using
Kubernetes. But if you're just running a few Docker hosts, it might seem like overkill.
### Tailscale
[Tailscale](https://tailscale.com/) makes WireGuard easy with automatic NAT traversal and key management, but it's not
designed as a generic site-to-site VPN for connecting networks. Instead, it connects individual devices and provides
identity-based access controls.
The recommended approach for using [Tailscale with Docker](https://tailscale.com/kb/1282/docker) is to connect each
individual container to a Tailscale network. This means deploying an additional Tailscale container alongside every
application container.
Tailscale's [subnet router](https://tailscale.com/kb/1019/subnets) feature might work to expose Docker networks similar
to our setup, but I haven't tested this approach.
//Not a generic site-to-site VPN, so the recommended approach is to use Tailscale on the container level. This way a
//container that needs to talk across machines is configured as a Tailscale machine so it can connect to other Tailscale
//machines. Maybe the subnet router feature can be used to connect Docker networks in a similar I described here, but
//I haven't tested it.
## Conclusion
That's it! Now you know how to securely connect Docker containers across multiple machines using WireGuard. The manual
setup works great for a handful of machines that you don't need to change often, but configuration management becomes
tedious as you scale.
//Summarise what we've done.?
If you don't want to mess with manual configuration, consider automation tools like Uncloud or evaluate if you need a
full orchestration platform.
Feel free to reach out if you have any questions or suggestions. You can find me on X
at [@psviderski](https://x.com/psviderski) or check my GitHub profile [psviderski](https://github.com/psviderski/)
for other contacts.
If you like this article and my work, you can follow me on X [@psviderski](https://x.com/psviderski).

Before

Width:  |  Height:  |  Size: 897 KiB

After

Width:  |  Height:  |  Size: 897 KiB

+8
View File
@@ -0,0 +1,8 @@
services:
uncloud-docs:
image: ghcr.io/psviderski/uncloud-docs:latest
pull_policy: always
user: nobody
x-ports:
- docs.uncloud.run:8000/https
scale: 2
View File
@@ -89,23 +89,6 @@ For example, move it to `/usr/local/bin` which is a common location for user-ins
sudo mv ./uc /usr/local/bin
```
Follow the same steps to upgrade to the latest version in the future.
## Debian
On a Debian system, you can install Uncloud CLI from an unofficial
[repository](https://debian.griffo.io/) maintained by
[@dariogriffo](https://github.com/dariogriffo):
```shell
curl -sS https://debian.griffo.io/EA0F721D231FDD3A0A17B9AC7808B4DD62C41256.asc | sudo gpg --dearmor --yes -o /etc/apt/trusted.gpg.d/debian.griffo.io.gpg
echo "deb https://debian.griffo.io/apt $(lsb_release -sc 2>/dev/null) main" | sudo tee /etc/apt/sources.list.d/debian.griffo.io.list
apt install -y uncloud
```
Alternatively, you can download `.deb` packages directly from the repository
[releases](https://github.com/dariogriffo/uncloud-debian/releases) page.
## Verify installation
After installation, verify that `uc` command is working:
@@ -114,6 +97,22 @@ After installation, verify that `uc` command is working:
uc --version
```
## Linux (via package managers)
### Debian
Via unofficial repository packages created and maintained at [uncloud-debian](https://github.com/dariogriffo/uncloud-debian/) by @dariogriffo
You can install uncloud the debian way by running:
```sh
curl -sS https://debian.griffo.io/EA0F721D231FDD3A0A17B9AC7808B4DD62C41256.asc | sudo gpg --dearmor --yes -o /etc/apt/trusted.gpg.d/debian.griffo.io.gpg
echo "deb https://debian.griffo.io/apt $(lsb_release -sc 2>/dev/null) main" | sudo tee /etc/apt/sources.list.d/debian.griffo.io.list
apt install -y uncloud
```
or in the releases page of the repository [here](https://github.com/dariogriffo/uncloud-debian/releases)
## Next steps
Now that you have `uc` installed, you're ready to:
@@ -9,8 +9,8 @@ infrastructure with secure internet access.
Before you begin, you'll need:
- **Uncloud CLI** [installed](1-install-cli.md) on your local machine
- A **Ubuntu or Debian server** with **public IP address** and **SSH access** using a **private key** (as `root` or a
user with **passwordless** `sudo` privileges).
- A **Ubuntu or Debian server** with **public IP address** and **SSH access** (as `root` or a user with `sudo`
privileges) using a **private key**.
:::tip Need a server?
@@ -271,16 +271,6 @@ Want to use your own domain, for example, `excalidraw.example.com` instead of `e
Add a CNAME record `excalidraw.example.com` in your DNS provider (Cloudflare, Namecheap, etc.) pointing to
`excalidraw.7za6s7.cluster.uncloud.run`. Alternatively, you can add an A record pointing to your server's IP.
:::info note
These instructions set up your own domain **in addition to** the Uncloud managed DNS name
`excalidraw.7za6s7.cluster.uncloud.run`.
If you want to avoid the managed service altogether, add `--no-dns` to your `uc machine init` command, and point an A
DNS record to your server(s)'s IP(s).
:::
Then update the published port `80/https` in `compose.yaml` to use your domain:
```yaml title="compose.yaml"

Before

Width:  |  Height:  |  Size: 389 KiB

After

Width:  |  Height:  |  Size: 389 KiB

@@ -10,12 +10,12 @@ import {themes as prismThemes} from 'prism-react-renderer';
/** @type {import('@docusaurus/types').Config} */
const config = {
title: 'Uncloud',
tagline: 'Self-host and scale web apps without Kubernetes complexity',
title: 'Uncloud Docs',
tagline: 'Dinosaurs are cool',
favicon: 'img/favicon.png',
// Set the production url of your site here
url: 'https://uncloud.run',
url: 'https://docs.uncloud.run',
// Set the /<baseUrl>/ pathname under which your site is served
// For GitHub pages deployment, it is often '/<projectName>/'
baseUrl: '/',
@@ -54,30 +54,27 @@ const config = {
({
docs: {
// Remove this to remove the "edit this page" links.
editUrl: 'https://github.com/psviderski/uncloud/edit/main/website/',
editUrl: 'https://github.com/psviderski/uncloud/edit/main/docs/',
// Serve the docs at the site's root.
// routeBasePath: '/',
routeBasePath: '/',
showLastUpdateTime: true,
sidebarPath: './sidebars.js',
},
blog: {
blogDescription: 'Blog posts from the Uncloud team and community',
blogSidebarTitle: 'All posts',
blogSidebarCount: 'ALL',
showReadingTime: true,
feedOptions: {
type: ['rss', 'atom'],
xslt: true,
},
// Please change this to your repo.
// Remove this to remove the "edit this page" links.
// blog: {
// showReadingTime: true,
// feedOptions: {
// type: ['rss', 'atom'],
// xslt: true,
// },
// // Please change this to your repo.
// // Remove this to remove the "edit this page" links.
// editUrl:
// 'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/',
// Useful options to enforce blogging best practices
onInlineTags: 'warn',
onInlineAuthors: 'warn',
onUntruncatedBlogPosts: 'warn',
},
// // Useful options to enforce blogging best practices
// onInlineTags: 'warn',
// onInlineAuthors: 'warn',
// onUntruncatedBlogPosts: 'warn',
// },
theme: {
customCss: './src/css/custom.css',
},
@@ -99,24 +96,12 @@ const config = {
// Relative to your site's "static" directory. Cannot be SVGs. Can be external URLs too.
image: 'img/social-card.png',
navbar: {
title: 'Uncloud',
title: 'Uncloud Docs',
logo: {
alt: 'Uncloud Logo',
src: 'img/logo.svg',
href: 'https://uncloud.run',
},
items: [
{
type: 'doc',
docId: 'overview',
label: 'Docs',
position: 'left',
},
{
to: 'blog',
label: 'Blog',
position: 'left',
},
{
type: 'search',
position: 'right',
+18434
View File
File diff suppressed because it is too large Load Diff
+45
View File
@@ -0,0 +1,45 @@
{
"name": "docs",
"version": "0.0.0",
"private": true,
"scripts": {
"docusaurus": "docusaurus",
"start": "docusaurus start",
"build": "docusaurus build",
"swizzle": "docusaurus swizzle",
"deploy": "docusaurus deploy",
"clear": "docusaurus clear",
"serve": "docusaurus serve",
"write-translations": "docusaurus write-translations",
"write-heading-ids": "docusaurus write-heading-ids"
},
"dependencies": {
"@docusaurus/core": "3.7.0",
"@docusaurus/preset-classic": "3.7.0",
"@easyops-cn/docusaurus-search-local": "^0.49.2",
"@mdx-js/react": "^3.0.0",
"clsx": "^2.0.0",
"prism-react-renderer": "^2.3.0",
"react": "^19.0.0",
"react-dom": "^19.0.0"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "3.7.0",
"@docusaurus/types": "3.7.0"
},
"browserslist": {
"production": [
">0.5%",
"not dead",
"not op_mini all"
],
"development": [
"last 3 chrome version",
"last 3 firefox version",
"last 5 safari version"
]
},
"engines": {
"node": ">=18.0"
}
}

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

Before

Width:  |  Height:  |  Size: 341 B

After

Width:  |  Height:  |  Size: 341 B

Before

Width:  |  Height:  |  Size: 88 KiB

After

Width:  |  Height:  |  Size: 88 KiB

+1 -2
View File
@@ -20,7 +20,7 @@ Uncloud stores its configuration in `~/.config/uncloud/config.yaml`. If you wish
### Initialisation
Begin by initialising the first node in your cluster with `uc machine init [USER@HOST:PORT]`. If you do not have a need for Caddy reverse proxy, you may disable this feature with `--no-caddy`. If you want to avoid using uncloud's managed DNS service, add the `--no-dns` flag.
Begin by initialising the first node in your cluster with `uc machine init [USER@HOST:PORT]`. If you do not have a need for Caddy reverse proxy, you may disable this feature with `--no-caddy`.
This command will idempotently install Docker, uncloudd, uncloud-corrosion. If Caddy is enabled, it will set up a reverse proxy. If Uncloud DNS is enabled, it will create a DNS A record for the machine's public IP address under `*.[CLUSTER ID].cluster.uncloud.run`.
@@ -37,7 +37,6 @@ Uncloud (uncloud.run) DNS can be managed with the `uc dns` subcommand.
* To reserve a domain name, run `uc dns reserve`
* To release a domain name, run `uc dns release`.
* To see the domain name, run `uc dns show`
* To avoid using the Uncloud managed DNS service, use the `--no-dns` flag on your `uc machine init` command.
### Running a service
+2 -3
View File
@@ -3,11 +3,10 @@ package main
import (
"context"
"fmt"
"log/slog"
"time"
"github.com/hashicorp/serf/serf"
crdt "github.com/ipfs/go-ds-crdt"
"log/slog"
"time"
)
// Implements the Broadcaster interface.
+1 -2
View File
@@ -2,10 +2,9 @@ package main
import (
"fmt"
"github.com/dgraph-io/badger/v3"
"log"
"time"
"github.com/dgraph-io/badger/v3"
)
func customTimeEncoder(t time.Time) string {
+1 -2
View File
@@ -2,10 +2,9 @@ package main
import (
"fmt"
"github.com/ipfs/go-log/v2"
"log/slog"
"os"
"github.com/ipfs/go-log/v2"
)
// ipfsLogger is an slog logger that implements the IPFS go-log StandardLogger interface.
+1 -2
View File
@@ -2,13 +2,12 @@ package main
import (
"fmt"
"github.com/docker/docker/libnetwork/networkdb"
"log/slog"
"os"
"os/signal"
"syscall"
"time"
"github.com/docker/docker/libnetwork/networkdb"
)
func main() {
+6 -7
View File
@@ -4,13 +4,6 @@ import (
"context"
"flag"
"fmt"
"log/slog"
"net"
"os"
"os/signal"
"syscall"
"time"
"github.com/hashicorp/memberlist"
"github.com/hashicorp/serf/cmd/serf/command/agent"
"github.com/hashicorp/serf/serf"
@@ -18,6 +11,12 @@ import (
badger "github.com/ipfs/go-ds-badger3"
crdt "github.com/ipfs/go-ds-crdt"
"github.com/lmittmann/tint"
"log/slog"
"net"
"os"
"os/signal"
"syscall"
"time"
)
func createSerfAgentConfig(name, bindAddr, rpcAddr, profile string) *agent.Config {
+5 -4
View File
@@ -4,19 +4,18 @@ import (
"context"
"errors"
"fmt"
"log/slog"
"github.com/hashicorp/serf/serf"
"github.com/ipfs/boxo/datastore/dshelp"
dag "github.com/ipfs/boxo/ipld/merkledag"
"github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
ipld "github.com/ipfs/go-ipld-format"
"log/slog"
)
// Implements the DAGService interface.
// TODO: implement SessionDAGService to optimize node fetching.
// TODO: persistentSerfDAG?
// TOOD: persistentSerfDAG?
type dagSyncer struct {
// Persistent storage for the nodes.
store ds.Datastore
@@ -51,7 +50,8 @@ func (d *dagSyncer) Get(ctx context.Context, cid cid.Cid) (ipld.Node, error) {
}
slog.Debug("Queried node from peers", "cid", cid, "deadline", query.Deadline())
for {
responded := false
for !responded {
select {
case resp, ok := <-query.ResponseCh():
if !ok {
@@ -63,6 +63,7 @@ func (d *dagSyncer) Get(ctx context.Context, cid cid.Cid) (ipld.Node, error) {
continue
}
slog.Debug("Received node from peer", "cid", cid, "peer", resp.From)
responded = true
query.Close()
node, err = nodeFromBytes(resp.Payload)
+3 -4
View File
@@ -6,13 +6,12 @@ import (
"crypto/cipher"
"encoding/hex"
"fmt"
"net/netip"
"time"
"github.com/psviderski/uncloud/internal/machine/network"
"github.com/siderolabs/discovery-api/api/v1alpha1/client/pb"
discovery "github.com/siderolabs/discovery-client/pkg/client"
"go.uber.org/zap"
"net/netip"
"time"
"github.com/psviderski/uncloud/internal/machine/network"
)
const (
+1 -1
View File
@@ -35,7 +35,6 @@ require (
github.com/jmoiron/sqlx v1.4.0
github.com/lmittmann/tint v1.0.5
github.com/miekg/dns v1.1.65
github.com/mitchellh/mapstructure v1.5.0
github.com/moby/term v0.5.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0
@@ -205,6 +204,7 @@ require (
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/buildkit v0.17.2 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
+35 -35
View File
@@ -6,8 +6,8 @@ import (
"fmt"
"net/netip"
"os"
"slices"
"github.com/charmbracelet/huh"
"github.com/docker/cli/cli/streams"
"github.com/psviderski/uncloud/internal/cli/config"
"github.com/psviderski/uncloud/internal/fs"
@@ -22,12 +22,7 @@ import (
"google.golang.org/protobuf/types/known/emptypb"
)
const (
// DefaultSSHKeyPath is the fallback location for the SSH private key when provisioning remote machines.
// Used when no key is explicitly provided and SSH agent authentication fails.
DefaultSSHKeyPath = "~/.ssh/id_ed25519"
defaultContextName = "default"
)
const defaultContextName = "default"
type CLI struct {
Config *config.Config
@@ -178,7 +173,7 @@ func (cli *CLI) initRemoteMachine(ctx context.Context, opts InitClusterOptions)
return nil, fmt.Errorf("cluster context '%s' already exists", contextName)
}
machineClient, err := provisionRemoteMachine(ctx, opts.RemoteMachine, opts.Version)
machineClient, err := cli.provisionRemoteMachine(ctx, *opts.RemoteMachine, opts.Version)
if err != nil {
return nil, err
}
@@ -195,7 +190,7 @@ func (cli *CLI) initRemoteMachine(ctx context.Context, opts InitClusterOptions)
return nil, fmt.Errorf("inspect machine: %w", err)
}
if minfo.Id != "" {
if err = promptResetMachine(ctx, machineClient.MachineClient); err != nil {
if err = cli.promptResetMachine(); err != nil {
return nil, err
}
}
@@ -254,7 +249,7 @@ type AddMachineOptions struct {
Context string
MachineName string
PublicIP *netip.Addr
RemoteMachine *RemoteMachine
RemoteMachine RemoteMachine
Version string
}
@@ -277,7 +272,7 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
}
}()
machineClient, err := provisionRemoteMachine(ctx, opts.RemoteMachine, opts.Version)
machineClient, err := cli.provisionRemoteMachine(ctx, opts.RemoteMachine, opts.Version)
if err != nil {
return nil, nil, err
}
@@ -293,18 +288,7 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
return nil, nil, fmt.Errorf("inspect machine: %w", err)
}
if minfo.Id != "" {
// Check if the machine is already a member of this cluster.
machines, err := c.ListMachines(ctx, nil)
if err != nil {
return nil, nil, fmt.Errorf("list cluster machines: %w", err)
}
if slices.ContainsFunc(machines, func(m *pb.MachineMember) bool {
return m.Machine.Id == minfo.Id
}) {
return nil, nil, fmt.Errorf("machine is already a member of this cluster (%s)", minfo.Name)
}
if err = promptResetMachine(ctx, machineClient.MachineClient); err != nil {
if err = cli.promptResetMachine(); err != nil {
return nil, nil, err
}
}
@@ -355,7 +339,7 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
return nil, nil, fmt.Errorf("add machine to cluster (context '%s'): %w", contextName, err)
}
// Get the most up-to-date list of other machines in the cluster to include them in the join request.
// List other machines in the cluster to include them in the join request.
machines, err := c.ListMachines(ctx, nil)
if err != nil {
return nil, nil, fmt.Errorf("list cluster machines: %w", err)
@@ -398,20 +382,11 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
// provisionRemoteMachine installs the Uncloud daemon and dependencies on the remote machine over SSH and returns
// a machine API client to interact with the machine. The client should be closed after use by the caller.
// The version parameter specifies the version of the Uncloud daemon to install. If empty, the latest version is used.
// The remoteMachine.SSHKeyPath could be updated to the default SSH key path if it is not set and the SSH agent
// authentication fails.
func provisionRemoteMachine(
ctx context.Context, remoteMachine *RemoteMachine, version string,
func (cli *CLI) provisionRemoteMachine(
ctx context.Context, remoteMachine RemoteMachine, version string,
) (*client.Client, error) {
// Provision the remote machine by installing the Uncloud daemon and dependencies over SSH.
sshClient, err := sshexec.Connect(remoteMachine.User, remoteMachine.Host, remoteMachine.Port, remoteMachine.KeyPath)
// If the SSH connection using SSH agent fails and no key path is provided, try to use the default SSH key.
if err != nil && remoteMachine.KeyPath == "" {
remoteMachine.KeyPath = DefaultSSHKeyPath
sshClient, err = sshexec.Connect(
remoteMachine.User, remoteMachine.Host, remoteMachine.Port, remoteMachine.KeyPath,
)
}
if err != nil {
return nil, fmt.Errorf(
"SSH login to remote machine %s: %w",
@@ -445,6 +420,31 @@ func provisionRemoteMachine(
return machineClient, nil
}
func (cli *CLI) promptResetMachine() error {
var confirm bool
form := huh.NewForm(
huh.NewGroup(
huh.NewConfirm().
Title(
"The remote machine is already initialised as a cluster member. Do you want to reset it first?",
).
Affirmative("Yes!").
Negative("No").
Value(&confirm),
),
).WithAccessible(true)
if err := form.Run(); err != nil {
return fmt.Errorf("prompt user to confirm: %w", err)
}
if !confirm {
return fmt.Errorf("remote machine is already initialised as a cluster member")
}
// TODO: implement resetting the remote machine.
return fmt.Errorf("resetting the remote machine is not implemented yet. " +
"Please manually run 'uncloud-uninstall' on the remote machine to fully uninstall Uncloud from it")
}
// ProgressOut returns an output stream for progress writer.
func (cli *CLI) ProgressOut() *streams.Out {
return streams.NewOut(os.Stdout)
+2 -2
View File
@@ -53,11 +53,11 @@ func (c *Config) Read() error {
func (c *Config) Save() error {
dir, _ := filepath.Split(c.path)
if err := os.MkdirAll(dir, 0o700); err != nil {
if err := os.MkdirAll(dir, 0700); err != nil {
return fmt.Errorf("create config directory '%s': %w", dir, err)
}
f, err := os.OpenFile(c.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
f, err := os.OpenFile(c.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return fmt.Errorf("write config file '%s': %w", c.path, err)
}
+2 -83
View File
@@ -5,20 +5,12 @@ import (
"fmt"
"os"
"strings"
"time"
"github.com/cenkalti/backoff/v4"
"github.com/charmbracelet/huh"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/sshexec"
"google.golang.org/protobuf/types/known/emptypb"
)
const (
// TODO: support pinning the script version to the CLI version.
installScriptURL = "https://raw.githubusercontent.com/psviderski/uncloud/refs/heads/main/scripts/install.sh"
rootUser = "root"
)
const installScriptURL = "https://raw.githubusercontent.com/psviderski/uncloud/refs/heads/main/scripts/install.sh"
type RemoteMachine struct {
User string
@@ -32,7 +24,7 @@ func installCmd(user string, version string) string {
var env []string
// Add the SSH user (non-root) to the uncloud group to allow access to the Uncloud daemon unix socket.
if user != rootUser {
if user != "root" {
sudoPrefix = "sudo"
env = append(env, "UNCLOUD_GROUP_ADD_USER="+sshexec.Quote(user))
}
@@ -54,26 +46,6 @@ func provisionMachine(ctx context.Context, exec sshexec.Executor, version string
return fmt.Errorf("run whoami: %w", err)
}
if user != rootUser {
// 'sudo -n' is not used because it fails with 'sudo: a password is required' when the user has no password
// in /etc/shadow even though it may have valid sudo access.
out, err := exec.Run(ctx, "sudo true")
if err != nil {
if strings.Contains(out, "password is required") {
return fmt.Errorf(
"user '%[1]s' requires a password for sudo, but Uncloud needs passwordless sudo or root access "+
"to install and configure the uncloudd daemon on the remote machine.\n\n"+
"Possible solutions:\n"+
"1. Use root user or a user with passwordless sudo instead.\n"+
"2. Configure passwordless sudo for the user '%[1]s' by running on the remote machine:\n"+
" echo '%[1]s ALL=(ALL) NOPASSWD:ALL' | sudo tee /etc/sudoers.d/%[1]s",
user)
}
return fmt.Errorf("sudo command failed for user '%s': %w. "+
"Please ensure the user has sudo privileges or use root user instead", user, err)
}
}
cmd := installCmd(user, version)
fmt.Println("Downloading Uncloud install script:", installScriptURL)
@@ -84,56 +56,3 @@ func provisionMachine(ctx context.Context, exec sshexec.Executor, version string
}
return nil
}
func promptResetMachine(ctx context.Context, machineClient pb.MachineClient) error {
var confirm bool
form := huh.NewForm(
huh.NewGroup(
huh.NewConfirm().
Title(
"The remote machine is already initialised as a cluster member. Do you want to reset it first?\n" +
"This will:\n" +
"- Remove all service containers from the machine\n" +
"- Reset the machine to the uninitialised state",
).
Affirmative("Yes!").
Negative("No").
Value(&confirm),
),
).WithAccessible(true)
if err := form.Run(); err != nil {
return fmt.Errorf("prompt user to confirm: %w", err)
}
if !confirm {
return fmt.Errorf("remote machine is already initialised as a cluster member")
}
if _, err := machineClient.Reset(ctx, &pb.ResetRequest{}); err != nil {
return fmt.Errorf("reset remote machine: %w. You can also manually run 'uncloud-uninstall' "+
"on the remote machine to fully uninstall Uncloud from it", err)
}
fmt.Println("Resetting the remote machine...")
if err := waitMachineReady(ctx, machineClient, 1*time.Minute); err != nil {
return fmt.Errorf("wait for machine to be ready after reset: %w", err)
}
return nil
}
// waitMachineReady waits for the machine to be ready to serve requests.
func waitMachineReady(ctx context.Context, machineClient pb.MachineClient, timeout time.Duration) error {
boff := backoff.WithContext(backoff.NewExponentialBackOff(
backoff.WithMaxInterval(1*time.Second),
backoff.WithMaxElapsedTime(timeout),
), ctx)
inspect := func() error {
_, err := machineClient.Inspect(ctx, &emptypb.Empty{})
if err != nil {
return fmt.Errorf("inspect machine: %w", err)
}
return nil
}
return backoff.Retry(inspect, boff)
}
+2 -3
View File
@@ -5,15 +5,14 @@ import (
"crypto/tls"
"errors"
"fmt"
"github.com/cenkalti/backoff/v4"
"golang.org/x/net/http2"
"log/slog"
"net"
"net/http"
"net/netip"
"net/url"
"time"
"github.com/cenkalti/backoff/v4"
"golang.org/x/net/http2"
)
const (
+1 -2
View File
@@ -6,12 +6,11 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/cenkalti/backoff/v4"
"io"
"log/slog"
"net/http"
"strconv"
"github.com/cenkalti/backoff/v4"
)
type ChangeType string
+1 -2
View File
@@ -3,9 +3,8 @@ package daemon
import (
"context"
"fmt"
"log/slog"
systemd "github.com/coreos/go-systemd/daemon"
"log/slog"
"github.com/psviderski/uncloud/internal/machine"
)
+2 -3
View File
@@ -3,11 +3,10 @@ package daemon
import (
"errors"
"fmt"
"net/netip"
"os"
"github.com/psviderski/uncloud/internal/machine"
"github.com/psviderski/uncloud/internal/machine/network"
"net/netip"
"os"
)
// MachineToken returns the local machine's token that can be used for adding the machine to a cluster.
+2 -3
View File
@@ -4,11 +4,10 @@ import (
"context"
"errors"
"fmt"
"log/slog"
"time"
"github.com/cenkalti/backoff/v4"
"github.com/docker/docker/client"
"log/slog"
"time"
)
// WaitDaemonReady waits for the Docker daemon to start and be ready to serve requests.
+107 -350
View File
@@ -124,7 +124,7 @@ func (x DNSRecord_RecordType) Number() protoreflect.EnumNumber {
// Deprecated: Use DNSRecord_RecordType.Descriptor instead.
func (DNSRecord_RecordType) EnumDescriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{11, 0}
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{8, 0}
}
type AddMachineRequest struct {
@@ -339,173 +339,6 @@ func (x *ListMachinesResponse) GetMachines() []*MachineMember {
return nil
}
type UpdateMachineRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Machine to update
MachineId string `protobuf:"bytes,1,opt,name=machine_id,json=machineId,proto3" json:"machine_id,omitempty"`
// Updated machine information
Name *string `protobuf:"bytes,2,opt,name=name,proto3,oneof" json:"name,omitempty"`
PublicIp *IP `protobuf:"bytes,3,opt,name=public_ip,json=publicIp,proto3,oneof" json:"public_ip,omitempty"`
Endpoints []*IPPort `protobuf:"bytes,4,rep,name=endpoints,proto3" json:"endpoints,omitempty"`
}
func (x *UpdateMachineRequest) Reset() {
*x = UpdateMachineRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *UpdateMachineRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*UpdateMachineRequest) ProtoMessage() {}
func (x *UpdateMachineRequest) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use UpdateMachineRequest.ProtoReflect.Descriptor instead.
func (*UpdateMachineRequest) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{4}
}
func (x *UpdateMachineRequest) GetMachineId() string {
if x != nil {
return x.MachineId
}
return ""
}
func (x *UpdateMachineRequest) GetName() string {
if x != nil && x.Name != nil {
return *x.Name
}
return ""
}
func (x *UpdateMachineRequest) GetPublicIp() *IP {
if x != nil {
return x.PublicIp
}
return nil
}
func (x *UpdateMachineRequest) GetEndpoints() []*IPPort {
if x != nil {
return x.Endpoints
}
return nil
}
type UpdateMachineResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Machine *MachineInfo `protobuf:"bytes,1,opt,name=machine,proto3" json:"machine,omitempty"`
}
func (x *UpdateMachineResponse) Reset() {
*x = UpdateMachineResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *UpdateMachineResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*UpdateMachineResponse) ProtoMessage() {}
func (x *UpdateMachineResponse) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[5]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use UpdateMachineResponse.ProtoReflect.Descriptor instead.
func (*UpdateMachineResponse) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{5}
}
func (x *UpdateMachineResponse) GetMachine() *MachineInfo {
if x != nil {
return x.Machine
}
return nil
}
type RemoveMachineRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
}
func (x *RemoveMachineRequest) Reset() {
*x = RemoveMachineRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *RemoveMachineRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*RemoveMachineRequest) ProtoMessage() {}
func (x *RemoveMachineRequest) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[6]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use RemoveMachineRequest.ProtoReflect.Descriptor instead.
func (*RemoveMachineRequest) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{6}
}
func (x *RemoveMachineRequest) GetId() string {
if x != nil {
return x.Id
}
return ""
}
type Domain struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -517,7 +350,7 @@ type Domain struct {
func (x *Domain) Reset() {
*x = Domain{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[7]
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -530,7 +363,7 @@ func (x *Domain) String() string {
func (*Domain) ProtoMessage() {}
func (x *Domain) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[7]
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -543,7 +376,7 @@ func (x *Domain) ProtoReflect() protoreflect.Message {
// Deprecated: Use Domain.ProtoReflect.Descriptor instead.
func (*Domain) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{7}
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{4}
}
func (x *Domain) GetName() string {
@@ -564,7 +397,7 @@ type ReserveDomainRequest struct {
func (x *ReserveDomainRequest) Reset() {
*x = ReserveDomainRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[8]
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -577,7 +410,7 @@ func (x *ReserveDomainRequest) String() string {
func (*ReserveDomainRequest) ProtoMessage() {}
func (x *ReserveDomainRequest) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[8]
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[5]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -590,7 +423,7 @@ func (x *ReserveDomainRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use ReserveDomainRequest.ProtoReflect.Descriptor instead.
func (*ReserveDomainRequest) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{8}
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{5}
}
func (x *ReserveDomainRequest) GetEndpoint() string {
@@ -611,7 +444,7 @@ type CreateDomainRecordsRequest struct {
func (x *CreateDomainRecordsRequest) Reset() {
*x = CreateDomainRecordsRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[9]
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -624,7 +457,7 @@ func (x *CreateDomainRecordsRequest) String() string {
func (*CreateDomainRecordsRequest) ProtoMessage() {}
func (x *CreateDomainRecordsRequest) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[9]
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[6]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -637,7 +470,7 @@ func (x *CreateDomainRecordsRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use CreateDomainRecordsRequest.ProtoReflect.Descriptor instead.
func (*CreateDomainRecordsRequest) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{9}
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{6}
}
func (x *CreateDomainRecordsRequest) GetRecords() []*DNSRecord {
@@ -658,7 +491,7 @@ type CreateDomainRecordsResponse struct {
func (x *CreateDomainRecordsResponse) Reset() {
*x = CreateDomainRecordsResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[10]
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -671,7 +504,7 @@ func (x *CreateDomainRecordsResponse) String() string {
func (*CreateDomainRecordsResponse) ProtoMessage() {}
func (x *CreateDomainRecordsResponse) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[10]
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[7]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -684,7 +517,7 @@ func (x *CreateDomainRecordsResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use CreateDomainRecordsResponse.ProtoReflect.Descriptor instead.
func (*CreateDomainRecordsResponse) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{10}
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{7}
}
func (x *CreateDomainRecordsResponse) GetRecords() []*DNSRecord {
@@ -707,7 +540,7 @@ type DNSRecord struct {
func (x *DNSRecord) Reset() {
*x = DNSRecord{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[11]
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -720,7 +553,7 @@ func (x *DNSRecord) String() string {
func (*DNSRecord) ProtoMessage() {}
func (x *DNSRecord) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[11]
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[8]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -733,7 +566,7 @@ func (x *DNSRecord) ProtoReflect() protoreflect.Message {
// Deprecated: Use DNSRecord.ProtoReflect.Descriptor instead.
func (*DNSRecord) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{11}
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{8}
}
func (x *DNSRecord) GetName() string {
@@ -797,87 +630,59 @@ var file_internal_machine_api_pb_cluster_proto_rawDesc = []byte{
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73,
0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63,
0x68, 0x69, 0x6e, 0x65, 0x4d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x52, 0x08, 0x6d, 0x61, 0x63, 0x68,
0x69, 0x6e, 0x65, 0x73, 0x22, 0xbb, 0x01, 0x0a, 0x14, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d,
0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a,
0x0a, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x09, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x04,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x04, 0x6e, 0x61,
0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x29, 0x0a, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f,
0x69, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x07, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49,
0x50, 0x48, 0x01, 0x52, 0x08, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x70, 0x88, 0x01, 0x01,
0x12, 0x29, 0x0a, 0x09, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x50, 0x50, 0x6f, 0x72, 0x74,
0x52, 0x09, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x42, 0x07, 0x0a, 0x05, 0x5f,
0x6e, 0x61, 0x6d, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f,
0x69, 0x70, 0x22, 0x43, 0x0a, 0x15, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68,
0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x07, 0x6d,
0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61,
0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07,
0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0x26, 0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f, 0x76,
0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22,
0x1c, 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d,
0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x32, 0x0a,
0x14, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
0x74, 0x22, 0x46, 0x0a, 0x1a, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69,
0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x28, 0x0a, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b,
0x32, 0x0e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x47, 0x0a, 0x1b, 0x43, 0x72, 0x65,
0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x07, 0x72, 0x65, 0x63, 0x6f,
0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x61, 0x70, 0x69, 0x2e,
0x44, 0x4e, 0x53, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72,
0x64, 0x73, 0x22, 0x96, 0x01, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01,
0x28, 0x0e, 0x32, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x63, 0x6f,
0x72, 0x64, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74,
0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x03, 0x20,
0x03, 0x28, 0x09, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x22, 0x2e, 0x0a, 0x0a, 0x52,
0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53,
0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x05, 0x0a, 0x01, 0x41, 0x10,
0x01, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x41, 0x41, 0x41, 0x10, 0x02, 0x32, 0x92, 0x04, 0x0a, 0x07,
0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x3d, 0x0a, 0x0a, 0x41, 0x64, 0x64, 0x4d, 0x61,
0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x41, 0x64, 0x64, 0x4d,
0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e,
0x61, 0x70, 0x69, 0x2e, 0x41, 0x64, 0x64, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61,
0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x19,
0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x0d, 0x55, 0x70, 0x64,
0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x19, 0x2e, 0x61, 0x70, 0x69,
0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x55, 0x70, 0x64, 0x61,
0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x12, 0x42, 0x0a, 0x0d, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69,
0x6e, 0x65, 0x12, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4d,
0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e,
0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x37, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65,
0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x65, 0x73,
0x65, 0x72, 0x76, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x30,
0x0a, 0x09, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f,
0x69, 0x6e, 0x65, 0x73, 0x22, 0x1c, 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x12,
0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61,
0x6d, 0x65, 0x22, 0x32, 0x0a, 0x14, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x44, 0x6f, 0x6d,
0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e,
0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e,
0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x22, 0x46, 0x0a, 0x1a, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65,
0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x12, 0x28, 0x0a, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18,
0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x4e, 0x53, 0x52,
0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x47,
0x0a, 0x1b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65,
0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a,
0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e,
0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07,
0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x96, 0x01, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x52,
0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70,
0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x4e,
0x53, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79,
0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75,
0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73,
0x22, 0x2e, 0x0a, 0x0a, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f,
0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12,
0x05, 0x0a, 0x01, 0x41, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x41, 0x41, 0x41, 0x10, 0x02,
0x32, 0x86, 0x03, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x3d, 0x0a, 0x0a,
0x41, 0x64, 0x64, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x16, 0x2e, 0x61, 0x70, 0x69,
0x2e, 0x41, 0x64, 0x64, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x17, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x41, 0x64, 0x64, 0x4d, 0x61, 0x63, 0x68,
0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0c, 0x4c,
0x69, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d,
0x70, 0x74, 0x79, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e,
0x12, 0x34, 0x0a, 0x0d, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69,
0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x2e,
0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x58, 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65,
0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x12, 0x1f, 0x2e,
0x61, 0x70, 0x69, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e,
0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20,
0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69,
0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x42, 0x37, 0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70,
0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x6b, 0x69, 0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75,
0x64, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6d, 0x61, 0x63, 0x68, 0x69,
0x6e, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x33,
0x70, 0x74, 0x79, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61,
0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37,
0x0a, 0x0d, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12,
0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x44, 0x6f, 0x6d,
0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69,
0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x30, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x44, 0x6f,
0x6d, 0x61, 0x69, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x0b, 0x2e, 0x61,
0x70, 0x69, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x34, 0x0a, 0x0d, 0x52, 0x65, 0x6c,
0x65, 0x61, 0x73, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70,
0x74, 0x79, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12,
0x58, 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52,
0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x12, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x72, 0x65,
0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x72,
0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x37, 0x5a, 0x35, 0x67, 0x69, 0x74,
0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73,
0x6b, 0x69, 0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72,
0x6e, 0x61, 0x6c, 0x2f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f,
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -893,7 +698,7 @@ func file_internal_machine_api_pb_cluster_proto_rawDescGZIP() []byte {
}
var file_internal_machine_api_pb_cluster_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
var file_internal_machine_api_pb_cluster_proto_msgTypes = make([]protoimpl.MessageInfo, 12)
var file_internal_machine_api_pb_cluster_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
var file_internal_machine_api_pb_cluster_proto_goTypes = []any{
(MachineMember_MembershipState)(0), // 0: api.MachineMember.MembershipState
(DNSRecord_RecordType)(0), // 1: api.DNSRecord.RecordType
@@ -901,54 +706,43 @@ var file_internal_machine_api_pb_cluster_proto_goTypes = []any{
(*AddMachineResponse)(nil), // 3: api.AddMachineResponse
(*MachineMember)(nil), // 4: api.MachineMember
(*ListMachinesResponse)(nil), // 5: api.ListMachinesResponse
(*UpdateMachineRequest)(nil), // 6: api.UpdateMachineRequest
(*UpdateMachineResponse)(nil), // 7: api.UpdateMachineResponse
(*RemoveMachineRequest)(nil), // 8: api.RemoveMachineRequest
(*Domain)(nil), // 9: api.Domain
(*ReserveDomainRequest)(nil), // 10: api.ReserveDomainRequest
(*CreateDomainRecordsRequest)(nil), // 11: api.CreateDomainRecordsRequest
(*CreateDomainRecordsResponse)(nil), // 12: api.CreateDomainRecordsResponse
(*DNSRecord)(nil), // 13: api.DNSRecord
(*NetworkConfig)(nil), // 14: api.NetworkConfig
(*IP)(nil), // 15: api.IP
(*MachineInfo)(nil), // 16: api.MachineInfo
(*IPPort)(nil), // 17: api.IPPort
(*emptypb.Empty)(nil), // 18: google.protobuf.Empty
(*Domain)(nil), // 6: api.Domain
(*ReserveDomainRequest)(nil), // 7: api.ReserveDomainRequest
(*CreateDomainRecordsRequest)(nil), // 8: api.CreateDomainRecordsRequest
(*CreateDomainRecordsResponse)(nil), // 9: api.CreateDomainRecordsResponse
(*DNSRecord)(nil), // 10: api.DNSRecord
(*NetworkConfig)(nil), // 11: api.NetworkConfig
(*IP)(nil), // 12: api.IP
(*MachineInfo)(nil), // 13: api.MachineInfo
(*emptypb.Empty)(nil), // 14: google.protobuf.Empty
}
var file_internal_machine_api_pb_cluster_proto_depIdxs = []int32{
14, // 0: api.AddMachineRequest.network:type_name -> api.NetworkConfig
15, // 1: api.AddMachineRequest.public_ip:type_name -> api.IP
16, // 2: api.AddMachineResponse.machine:type_name -> api.MachineInfo
16, // 3: api.MachineMember.machine:type_name -> api.MachineInfo
11, // 0: api.AddMachineRequest.network:type_name -> api.NetworkConfig
12, // 1: api.AddMachineRequest.public_ip:type_name -> api.IP
13, // 2: api.AddMachineResponse.machine:type_name -> api.MachineInfo
13, // 3: api.MachineMember.machine:type_name -> api.MachineInfo
0, // 4: api.MachineMember.state:type_name -> api.MachineMember.MembershipState
4, // 5: api.ListMachinesResponse.machines:type_name -> api.MachineMember
15, // 6: api.UpdateMachineRequest.public_ip:type_name -> api.IP
17, // 7: api.UpdateMachineRequest.endpoints:type_name -> api.IPPort
16, // 8: api.UpdateMachineResponse.machine:type_name -> api.MachineInfo
13, // 9: api.CreateDomainRecordsRequest.records:type_name -> api.DNSRecord
13, // 10: api.CreateDomainRecordsResponse.records:type_name -> api.DNSRecord
1, // 11: api.DNSRecord.type:type_name -> api.DNSRecord.RecordType
2, // 12: api.Cluster.AddMachine:input_type -> api.AddMachineRequest
18, // 13: api.Cluster.ListMachines:input_type -> google.protobuf.Empty
6, // 14: api.Cluster.UpdateMachine:input_type -> api.UpdateMachineRequest
8, // 15: api.Cluster.RemoveMachine:input_type -> api.RemoveMachineRequest
10, // 16: api.Cluster.ReserveDomain:input_type -> api.ReserveDomainRequest
18, // 17: api.Cluster.GetDomain:input_type -> google.protobuf.Empty
18, // 18: api.Cluster.ReleaseDomain:input_type -> google.protobuf.Empty
11, // 19: api.Cluster.CreateDomainRecords:input_type -> api.CreateDomainRecordsRequest
3, // 20: api.Cluster.AddMachine:output_type -> api.AddMachineResponse
5, // 21: api.Cluster.ListMachines:output_type -> api.ListMachinesResponse
7, // 22: api.Cluster.UpdateMachine:output_type -> api.UpdateMachineResponse
18, // 23: api.Cluster.RemoveMachine:output_type -> google.protobuf.Empty
9, // 24: api.Cluster.ReserveDomain:output_type -> api.Domain
9, // 25: api.Cluster.GetDomain:output_type -> api.Domain
9, // 26: api.Cluster.ReleaseDomain:output_type -> api.Domain
12, // 27: api.Cluster.CreateDomainRecords:output_type -> api.CreateDomainRecordsResponse
20, // [20:28] is the sub-list for method output_type
12, // [12:20] is the sub-list for method input_type
12, // [12:12] is the sub-list for extension type_name
12, // [12:12] is the sub-list for extension extendee
0, // [0:12] is the sub-list for field type_name
10, // 6: api.CreateDomainRecordsRequest.records:type_name -> api.DNSRecord
10, // 7: api.CreateDomainRecordsResponse.records:type_name -> api.DNSRecord
1, // 8: api.DNSRecord.type:type_name -> api.DNSRecord.RecordType
2, // 9: api.Cluster.AddMachine:input_type -> api.AddMachineRequest
14, // 10: api.Cluster.ListMachines:input_type -> google.protobuf.Empty
7, // 11: api.Cluster.ReserveDomain:input_type -> api.ReserveDomainRequest
14, // 12: api.Cluster.GetDomain:input_type -> google.protobuf.Empty
14, // 13: api.Cluster.ReleaseDomain:input_type -> google.protobuf.Empty
8, // 14: api.Cluster.CreateDomainRecords:input_type -> api.CreateDomainRecordsRequest
3, // 15: api.Cluster.AddMachine:output_type -> api.AddMachineResponse
5, // 16: api.Cluster.ListMachines:output_type -> api.ListMachinesResponse
6, // 17: api.Cluster.ReserveDomain:output_type -> api.Domain
6, // 18: api.Cluster.GetDomain:output_type -> api.Domain
6, // 19: api.Cluster.ReleaseDomain:output_type -> api.Domain
9, // 20: api.Cluster.CreateDomainRecords:output_type -> api.CreateDomainRecordsResponse
15, // [15:21] is the sub-list for method output_type
9, // [9:15] is the sub-list for method input_type
9, // [9:9] is the sub-list for extension type_name
9, // [9:9] is the sub-list for extension extendee
0, // [0:9] is the sub-list for field type_name
}
func init() { file_internal_machine_api_pb_cluster_proto_init() }
@@ -1008,42 +802,6 @@ func file_internal_machine_api_pb_cluster_proto_init() {
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[4].Exporter = func(v any, i int) any {
switch v := v.(*UpdateMachineRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[5].Exporter = func(v any, i int) any {
switch v := v.(*UpdateMachineResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[6].Exporter = func(v any, i int) any {
switch v := v.(*RemoveMachineRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[7].Exporter = func(v any, i int) any {
switch v := v.(*Domain); i {
case 0:
return &v.state
@@ -1055,7 +813,7 @@ func file_internal_machine_api_pb_cluster_proto_init() {
return nil
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[8].Exporter = func(v any, i int) any {
file_internal_machine_api_pb_cluster_proto_msgTypes[5].Exporter = func(v any, i int) any {
switch v := v.(*ReserveDomainRequest); i {
case 0:
return &v.state
@@ -1067,7 +825,7 @@ func file_internal_machine_api_pb_cluster_proto_init() {
return nil
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[9].Exporter = func(v any, i int) any {
file_internal_machine_api_pb_cluster_proto_msgTypes[6].Exporter = func(v any, i int) any {
switch v := v.(*CreateDomainRecordsRequest); i {
case 0:
return &v.state
@@ -1079,7 +837,7 @@ func file_internal_machine_api_pb_cluster_proto_init() {
return nil
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[10].Exporter = func(v any, i int) any {
file_internal_machine_api_pb_cluster_proto_msgTypes[7].Exporter = func(v any, i int) any {
switch v := v.(*CreateDomainRecordsResponse); i {
case 0:
return &v.state
@@ -1091,7 +849,7 @@ func file_internal_machine_api_pb_cluster_proto_init() {
return nil
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[11].Exporter = func(v any, i int) any {
file_internal_machine_api_pb_cluster_proto_msgTypes[8].Exporter = func(v any, i int) any {
switch v := v.(*DNSRecord); i {
case 0:
return &v.state
@@ -1104,14 +862,13 @@ func file_internal_machine_api_pb_cluster_proto_init() {
}
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[4].OneofWrappers = []any{}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_internal_machine_api_pb_cluster_proto_rawDesc,
NumEnums: 2,
NumMessages: 12,
NumMessages: 9,
NumExtensions: 0,
NumServices: 1,
},
-20
View File
@@ -11,8 +11,6 @@ import "internal/machine/api/pb/machine.proto";
service Cluster {
rpc AddMachine(AddMachineRequest) returns (AddMachineResponse);
rpc ListMachines(google.protobuf.Empty) returns (ListMachinesResponse);
rpc UpdateMachine(UpdateMachineRequest) returns (UpdateMachineResponse);
rpc RemoveMachine(RemoveMachineRequest) returns (google.protobuf.Empty);
rpc ReserveDomain(ReserveDomainRequest) returns (Domain);
rpc GetDomain(google.protobuf.Empty) returns (Domain);
@@ -51,24 +49,6 @@ message ListMachinesResponse {
repeated MachineMember machines = 1;
}
message UpdateMachineRequest {
// Machine to update
string machine_id = 1;
// Updated machine information
optional string name = 2;
optional IP public_ip = 3;
repeated IPPort endpoints = 4;
}
message UpdateMachineResponse {
MachineInfo machine = 1;
}
message RemoveMachineRequest {
string id = 1;
}
message Domain {
string name = 1;
}
@@ -22,8 +22,6 @@ const _ = grpc.SupportPackageIsVersion9
const (
Cluster_AddMachine_FullMethodName = "/api.Cluster/AddMachine"
Cluster_ListMachines_FullMethodName = "/api.Cluster/ListMachines"
Cluster_UpdateMachine_FullMethodName = "/api.Cluster/UpdateMachine"
Cluster_RemoveMachine_FullMethodName = "/api.Cluster/RemoveMachine"
Cluster_ReserveDomain_FullMethodName = "/api.Cluster/ReserveDomain"
Cluster_GetDomain_FullMethodName = "/api.Cluster/GetDomain"
Cluster_ReleaseDomain_FullMethodName = "/api.Cluster/ReleaseDomain"
@@ -36,8 +34,6 @@ const (
type ClusterClient interface {
AddMachine(ctx context.Context, in *AddMachineRequest, opts ...grpc.CallOption) (*AddMachineResponse, error)
ListMachines(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ListMachinesResponse, error)
UpdateMachine(ctx context.Context, in *UpdateMachineRequest, opts ...grpc.CallOption) (*UpdateMachineResponse, error)
RemoveMachine(ctx context.Context, in *RemoveMachineRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
ReserveDomain(ctx context.Context, in *ReserveDomainRequest, opts ...grpc.CallOption) (*Domain, error)
GetDomain(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*Domain, error)
ReleaseDomain(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*Domain, error)
@@ -72,26 +68,6 @@ func (c *clusterClient) ListMachines(ctx context.Context, in *emptypb.Empty, opt
return out, nil
}
func (c *clusterClient) UpdateMachine(ctx context.Context, in *UpdateMachineRequest, opts ...grpc.CallOption) (*UpdateMachineResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(UpdateMachineResponse)
err := c.cc.Invoke(ctx, Cluster_UpdateMachine_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *clusterClient) RemoveMachine(ctx context.Context, in *RemoveMachineRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(emptypb.Empty)
err := c.cc.Invoke(ctx, Cluster_RemoveMachine_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *clusterClient) ReserveDomain(ctx context.Context, in *ReserveDomainRequest, opts ...grpc.CallOption) (*Domain, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(Domain)
@@ -138,8 +114,6 @@ func (c *clusterClient) CreateDomainRecords(ctx context.Context, in *CreateDomai
type ClusterServer interface {
AddMachine(context.Context, *AddMachineRequest) (*AddMachineResponse, error)
ListMachines(context.Context, *emptypb.Empty) (*ListMachinesResponse, error)
UpdateMachine(context.Context, *UpdateMachineRequest) (*UpdateMachineResponse, error)
RemoveMachine(context.Context, *RemoveMachineRequest) (*emptypb.Empty, error)
ReserveDomain(context.Context, *ReserveDomainRequest) (*Domain, error)
GetDomain(context.Context, *emptypb.Empty) (*Domain, error)
ReleaseDomain(context.Context, *emptypb.Empty) (*Domain, error)
@@ -160,12 +134,6 @@ func (UnimplementedClusterServer) AddMachine(context.Context, *AddMachineRequest
func (UnimplementedClusterServer) ListMachines(context.Context, *emptypb.Empty) (*ListMachinesResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListMachines not implemented")
}
func (UnimplementedClusterServer) UpdateMachine(context.Context, *UpdateMachineRequest) (*UpdateMachineResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method UpdateMachine not implemented")
}
func (UnimplementedClusterServer) RemoveMachine(context.Context, *RemoveMachineRequest) (*emptypb.Empty, error) {
return nil, status.Errorf(codes.Unimplemented, "method RemoveMachine not implemented")
}
func (UnimplementedClusterServer) ReserveDomain(context.Context, *ReserveDomainRequest) (*Domain, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReserveDomain not implemented")
}
@@ -235,42 +203,6 @@ func _Cluster_ListMachines_Handler(srv interface{}, ctx context.Context, dec fun
return interceptor(ctx, in, info, handler)
}
func _Cluster_UpdateMachine_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(UpdateMachineRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ClusterServer).UpdateMachine(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: Cluster_UpdateMachine_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ClusterServer).UpdateMachine(ctx, req.(*UpdateMachineRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Cluster_RemoveMachine_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RemoveMachineRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ClusterServer).RemoveMachine(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: Cluster_RemoveMachine_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ClusterServer).RemoveMachine(ctx, req.(*RemoveMachineRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Cluster_ReserveDomain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReserveDomainRequest)
if err := dec(in); err != nil {
@@ -358,14 +290,6 @@ var Cluster_ServiceDesc = grpc.ServiceDesc{
MethodName: "ListMachines",
Handler: _Cluster_ListMachines_Handler,
},
{
MethodName: "UpdateMachine",
Handler: _Cluster_UpdateMachine_Handler,
},
{
MethodName: "RemoveMachine",
Handler: _Cluster_RemoveMachine_Handler,
},
{
MethodName: "ReserveDomain",
Handler: _Cluster_ReserveDomain_Handler,
+1 -1
View File
@@ -14,7 +14,7 @@ service Machine {
rpc JoinCluster(JoinClusterRequest) returns (google.protobuf.Empty);
rpc Token(google.protobuf.Empty) returns (TokenResponse);
rpc Inspect(google.protobuf.Empty) returns (MachineInfo);
// Reset restores the machine to a clean state, removing all cluster-related configuration and data.
// Reset restores the machine to a clean state, removing all cluster-related сonfiguration and data.
rpc Reset(ResetRequest) returns (google.protobuf.Empty);
rpc InspectService(InspectServiceRequest) returns (InspectServiceResponse);
+2 -2
View File
@@ -39,7 +39,7 @@ type MachineClient interface {
JoinCluster(ctx context.Context, in *JoinClusterRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
Token(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*TokenResponse, error)
Inspect(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*MachineInfo, error)
// Reset restores the machine to a clean state, removing all cluster-related configuration and data.
// Reset restores the machine to a clean state, removing all cluster-related сonfiguration and data.
Reset(ctx context.Context, in *ResetRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
InspectService(ctx context.Context, in *InspectServiceRequest, opts ...grpc.CallOption) (*InspectServiceResponse, error)
}
@@ -132,7 +132,7 @@ type MachineServer interface {
JoinCluster(context.Context, *JoinClusterRequest) (*emptypb.Empty, error)
Token(context.Context, *emptypb.Empty) (*TokenResponse, error)
Inspect(context.Context, *emptypb.Empty) (*MachineInfo, error)
// Reset restores the machine to a clean state, removing all cluster-related configuration and data.
// Reset restores the machine to a clean state, removing all cluster-related сonfiguration and data.
Reset(context.Context, *ResetRequest) (*emptypb.Empty, error)
InspectService(context.Context, *InspectServiceRequest) (*InspectServiceResponse, error)
mustEmbedUnimplementedMachineServer()
+1 -2
View File
@@ -2,11 +2,10 @@ package proxy
import (
"fmt"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/encoding/protowire"
"google.golang.org/protobuf/proto"
"github.com/psviderski/uncloud/internal/machine/api/pb"
)
// One2ManyResponder converts upstream responses into messages from upstreams, so that multiple
+1 -2
View File
@@ -2,12 +2,11 @@ package proxy
import (
"context"
"sync"
"github.com/siderolabs/grpc-proxy/proxy"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"sync"
)
// Director manages routing of gRPC requests between local and remote backends.
+1 -2
View File
@@ -2,12 +2,11 @@ package proxy
import (
"context"
"sync"
"github.com/siderolabs/grpc-proxy/proxy"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/metadata"
"sync"
)
// LocalBackend is a proxy.One2ManyResponder implementation that proxies to a local gRPC server listening on a Unix socket.
+3 -4
View File
@@ -3,15 +3,14 @@ package proxy
import (
"context"
"fmt"
"net/netip"
"sync"
"time"
"github.com/siderolabs/grpc-proxy/proxy"
"google.golang.org/grpc"
"google.golang.org/grpc/backoff"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/metadata"
"net/netip"
"sync"
"time"
)
// RemoteBackend is a proxy.One2ManyResponder implementation that proxies to a remote gRPC server, injecting machine metadata
-17
View File
@@ -1,17 +0,0 @@
package caddyconfig
import (
"fmt"
"github.com/psviderski/uncloud/pkg/api"
)
func GenerateCaddyfile(containers []api.ServiceContainer, verifyResponse string) (string, error) {
return fmt.Sprintf(`http:// {
handle %s {
respond "%s" 200
}
log
}
`, VerifyPath, verifyResponse), nil
}
@@ -19,7 +19,7 @@ import (
"github.com/psviderski/uncloud/pkg/api"
)
func GenerateJSONConfig(containers []api.ServiceContainer, verifyResponse string) (*caddy.Config, error) {
func GenerateConfig(containers []api.ServiceContainer, verifyResponse string) (*caddy.Config, error) {
// Maps hostnames to lists of upstreams (container IP:port pairs).
httpHostUpstreams := make(map[string][]string)
httpsHostUpstreams := make(map[string][]string)
@@ -378,7 +378,7 @@ func TestGenerateConfig(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config, err := GenerateJSONConfig(tt.containers, "verification-response-body")
config, err := GenerateConfig(tt.containers, "verification-response-body")
if tt.wantErr {
assert.Error(t, err)
+27 -51
View File
@@ -23,24 +23,23 @@ const (
// network.
type Controller struct {
store *store.Store
configDir string
path string
verifyResponse string
log *slog.Logger
}
func NewController(store *store.Store, configDir string, verifyResponse string) (*Controller, error) {
if err := os.MkdirAll(configDir, 0o750); err != nil {
return nil, fmt.Errorf("create directory for Caddy configuration '%s': %w", configDir, err)
func NewController(store *store.Store, path string, verifyResponse string) (*Controller, error) {
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0750); err != nil {
return nil, fmt.Errorf("create parent directory for Caddy configuration '%s': %w", dir, err)
}
if err := fs.Chown(configDir, "", CaddyGroup); err != nil {
return nil, fmt.Errorf("change owner of directory for Caddy configuration '%s': %w", configDir, err)
if err := fs.Chown(dir, "", CaddyGroup); err != nil {
return nil, fmt.Errorf("change owner of parent directory for Caddy configuration '%s': %w", dir, err)
}
return &Controller{
store: store,
configDir: configDir,
path: path,
verifyResponse: verifyResponse,
log: slog.With("component", "caddy-controller"),
}, nil
}
@@ -49,18 +48,14 @@ func (c *Controller) Run(ctx context.Context) error {
if err != nil {
return fmt.Errorf("subscribe to container changes: %w", err)
}
c.log.Info("Subscribed to container changes in the cluster to generate Caddy configuration.")
slog.Info("Subscribed to container changes in the cluster to generate Caddy configuration.")
containers, err := c.filterAvailableContainers(containerRecords)
if err != nil {
return fmt.Errorf("filter available containers: %w", err)
}
if err = c.generateCaddyfile(containers); err != nil {
return fmt.Errorf("generate Caddyfile configuration: %w", err)
}
if err = c.generateJSONConfig(containers); err != nil {
return fmt.Errorf("generate Caddy JSON configuration: %w", err)
if err = c.generateConfig(containers); err != nil {
return fmt.Errorf("generate Caddy configuration: %w", err)
}
for {
@@ -69,27 +64,23 @@ func (c *Controller) Run(ctx context.Context) error {
if !ok {
return fmt.Errorf("containers subscription failed")
}
c.log.Info("Cluster containers changed, updating Caddy configuration.")
slog.Debug("Cluster containers changed, updating Caddy configuration.")
containerRecords, err = c.store.ListContainers(ctx, store.ListOptions{})
if err != nil {
c.log.Info("Failed to list containers.", "err", err)
slog.Error("Failed to list containers.", "err", err)
continue
}
containers, err = c.filterAvailableContainers(containerRecords)
if err != nil {
c.log.Info("Failed to filter available containers.", "err", err)
slog.Error("Failed to filter available containers.", "err", err)
continue
}
if err = c.generateCaddyfile(containers); err != nil {
c.log.Info("Failed to generate Caddyfile configuration.", "err", err)
}
if err = c.generateJSONConfig(containers); err != nil {
c.log.Info("Failed to generate Caddy JSON configuration.", "err", err)
if err = c.generateConfig(containers); err != nil {
slog.Error("Failed to generate Caddy configuration.", "err", err)
}
c.log.Info("Updated Caddy configuration.", "dir", c.configDir)
slog.Debug("Updated Caddy configuration.", "path", c.path)
case <-ctx.Done():
return nil
}
@@ -104,30 +95,16 @@ func (c *Controller) filterAvailableContainers(
) ([]api.ServiceContainer, error) {
containers := make([]api.ServiceContainer, len(containerRecords))
for i, cr := range containerRecords {
containers[i] = cr.Container
containers[i] = api.ServiceContainer{
Container: cr.Container,
// TODO: restore ServiceSpec from the container record once it's saved in the store.
}
}
return containers, nil
}
func (c *Controller) generateCaddyfile(containers []api.ServiceContainer) error {
caddyfile, err := GenerateCaddyfile(containers, c.verifyResponse)
if err != nil {
return fmt.Errorf("generate Caddyfile: %w", err)
}
caddyfilePath := filepath.Join(c.configDir, "Caddyfile")
if err = os.WriteFile(caddyfilePath, []byte(caddyfile), 0o640); err != nil {
return fmt.Errorf("write Caddyfile to file '%s': %w", caddyfilePath, err)
}
if err = fs.Chown(caddyfilePath, "", CaddyGroup); err != nil {
return fmt.Errorf("change owner of Caddyfile '%s': %w", caddyfilePath, err)
}
return nil
}
func (c *Controller) generateJSONConfig(containers []api.ServiceContainer) error {
config, err := GenerateJSONConfig(containers, c.verifyResponse)
func (c *Controller) generateConfig(containers []api.ServiceContainer) error {
config, err := GenerateConfig(containers, c.verifyResponse)
if err != nil {
return err
}
@@ -136,13 +113,12 @@ func (c *Controller) generateJSONConfig(containers []api.ServiceContainer) error
if err != nil {
return fmt.Errorf("marshal Caddy configuration: %w", err)
}
configPath := filepath.Join(c.configDir, "caddy.json")
if err = os.WriteFile(configPath, configBytes, 0o640); err != nil {
return fmt.Errorf("write Caddy configuration to file '%s': %w", configPath, err)
if err = os.WriteFile(c.path, configBytes, 0640); err != nil {
return fmt.Errorf("write Caddy configuration to file '%s': %w", c.path, err)
}
if err = fs.Chown(configPath, "", CaddyGroup); err != nil {
return fmt.Errorf("change owner of Caddy configuration file '%s': %w", configPath, err)
if err = fs.Chown(c.path, "", CaddyGroup); err != nil {
return fmt.Errorf("change owner of Caddy configuration file '%s': %w", c.path, err)
}
return nil
+3 -108
View File
@@ -5,18 +5,17 @@ import (
"context"
"errors"
"fmt"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/emptypb"
"log/slog"
"net/netip"
"time"
"github.com/psviderski/uncloud/internal/corrosion"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/network"
"github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/internal/secret"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/emptypb"
)
type Cluster struct {
@@ -198,89 +197,6 @@ func (c *Cluster) AddMachine(ctx context.Context, req *pb.AddMachineRequest) (*p
return resp, nil
}
// UpdateMachine updates machine configuration in the cluster.
func (c *Cluster) UpdateMachine(ctx context.Context, req *pb.UpdateMachineRequest) (*pb.UpdateMachineResponse, error) {
if err := c.checkInitialised(ctx); err != nil {
return nil, err
}
if req.MachineId == "" {
return nil, status.Error(codes.InvalidArgument, "machine_id not set")
}
// Get the current machine info
currentMachine, err := c.store.GetMachine(ctx, req.MachineId)
if err != nil {
if errors.Is(err, store.ErrMachineNotFound) {
return nil, status.Errorf(codes.NotFound, "machine not found: %s", req.MachineId)
}
return nil, status.Errorf(codes.Internal, "failed to get machine: %v", err)
}
// Create a copy of the current machine for updating
updatedMachine := &pb.MachineInfo{
Id: currentMachine.Id,
Name: currentMachine.Name,
Network: currentMachine.Network,
PublicIp: currentMachine.PublicIp,
}
// Apply updates from the request
if req.Name != nil {
// Check for empty name
if *req.Name == "" {
return nil, status.Error(codes.InvalidArgument, "machine name cannot be empty")
}
// Check for duplicate names (excluding the current machine)
if *req.Name != currentMachine.Name {
machines, err := c.store.ListMachines(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "list machines: %v", err)
}
for _, m := range machines {
if m.Id != req.MachineId && m.Name == *req.Name {
return nil, status.Errorf(codes.AlreadyExists, "machine with name %q already exists", *req.Name)
}
}
}
updatedMachine.Name = *req.Name
}
if req.PublicIp != nil {
// Check if this is an empty IP (used to signal removal)
if len(req.PublicIp.Ip) == 0 {
// User wants to remove public IP
updatedMachine.PublicIp = nil
} else {
// Validate and set the new IP
ip, err := req.PublicIp.ToAddr()
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid public IP: %v", err)
}
if !ip.IsValid() {
return nil, status.Error(codes.InvalidArgument, "invalid public IP")
}
updatedMachine.PublicIp = req.PublicIp
}
}
if req.Endpoints != nil {
updatedMachine.Network.Endpoints = req.Endpoints
}
// Update the machine in the store
if err = c.store.UpdateMachine(ctx, updatedMachine); err != nil {
if errors.Is(err, store.ErrMachineNotFound) {
return nil, status.Errorf(codes.NotFound, "machine not found: %s", req.MachineId)
}
return nil, status.Errorf(codes.Internal, "update machine: %v", err)
}
slog.Info("Machine configuration updated in the cluster.",
"id", updatedMachine.Id, "name", updatedMachine.Name)
resp := &pb.UpdateMachineResponse{Machine: updatedMachine}
return resp, nil
}
// ListMachines lists all machines in the cluster including their membership states.
func (c *Cluster) ListMachines(ctx context.Context, _ *emptypb.Empty) (*pb.ListMachinesResponse, error) {
if err := c.checkInitialised(ctx); err != nil {
@@ -326,24 +242,3 @@ func (c *Cluster) ListMachines(ctx context.Context, _ *emptypb.Empty) (*pb.ListM
return &pb.ListMachinesResponse{Machines: members}, nil
}
// RemoveMachine removes a machine from the cluster.
func (c *Cluster) RemoveMachine(ctx context.Context, req *pb.RemoveMachineRequest) (*emptypb.Empty, error) {
if err := c.checkInitialised(ctx); err != nil {
return nil, err
}
if req.Id == "" {
return nil, status.Error(codes.InvalidArgument, "machine ID not set")
}
if err := c.store.DeleteMachine(ctx, req.Id); err != nil {
if errors.Is(err, store.ErrMachineNotFound) {
return nil, status.Errorf(codes.NotFound, "machine not found: %s", req.Id)
}
return nil, status.Errorf(codes.Internal, "delete machine from store: %v", err)
}
slog.Info("Machine removed from the cluster.", "id", req.Id)
return &emptypb.Empty{}, nil
}
-1
View File
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"errors"
"github.com/psviderski/uncloud/internal/dns"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/store"
+1 -2
View File
@@ -3,9 +3,8 @@ package cluster
import (
"errors"
"fmt"
"net/netip"
"go4.org/netipx"
"net/netip"
)
const DefaultSubnetBits = 24
-1
View File
@@ -2,7 +2,6 @@ package cluster
import (
"fmt"
"github.com/psviderski/uncloud/internal/secret"
)
-6
View File
@@ -1,6 +0,0 @@
package constants
const (
// MachineAPIPort is the port for the Machine API service on the management WireGuard network.
MachineAPIPort = 51000
)
+4 -5
View File
@@ -3,11 +3,10 @@ package corroservice
import (
"bytes"
"fmt"
"github.com/BurntSushi/toml"
"net/netip"
"os"
"path/filepath"
"github.com/BurntSushi/toml"
"github.com/psviderski/uncloud/internal/fs"
)
@@ -51,7 +50,7 @@ func (c *Config) Write(path, owner string) error {
if err := encoder.Encode(c); err != nil {
return fmt.Errorf("encode config: %w", err)
}
if err := os.WriteFile(path, data.Bytes(), 0o600); err != nil {
if err := os.WriteFile(path, data.Bytes(), 0600); err != nil {
return err
}
if err := fs.Chown(path, owner, owner); err != nil {
@@ -63,10 +62,10 @@ func (c *Config) Write(path, owner string) error {
func MkDataDir(dir, owner string) error {
parent, _ := filepath.Split(dir)
// Use 0711 for parent directories to allow `owner` to access its nested data directory.
if err := os.MkdirAll(parent, 0o711); err != nil {
if err := os.MkdirAll(parent, 0711); err != nil {
return fmt.Errorf("create directory %q: %w", parent, err)
}
if err := os.Mkdir(dir, 0o700); err != nil {
if err := os.Mkdir(dir, 0700); err != nil {
if !os.IsExist(err) {
return fmt.Errorf("create directory %q: %w", dir, err)
}
+17 -36
View File
@@ -3,16 +3,15 @@ package corroservice
import (
"context"
"fmt"
"io"
"log/slog"
"path/filepath"
"time"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"io"
"log/slog"
"path/filepath"
"time"
)
const (
@@ -27,47 +26,29 @@ type DockerService struct {
User string
}
func NewDockerService(cli *client.Client, image, name, dataDir string) *DockerService {
return &DockerService{
Client: cli,
Image: image,
Name: name,
DataDir: dataDir,
}
}
func (s *DockerService) Start(ctx context.Context) error {
_, err := s.Client.ContainerInspect(ctx, s.Name)
if err != nil {
if !client.IsErrNotFound(err) {
if client.IsErrNotFound(err) {
return s.startNewContainer(ctx)
}
return fmt.Errorf("inspect container %q: %w", s.Name, err)
}
if err = s.startNewContainer(ctx); err != nil {
return err
}
} else {
// Container already exists.
// TODO: recreate only if the container configuration has to be changed.
if err = s.Client.ContainerRemove(ctx, s.Name, container.RemoveOptions{Force: true}); err != nil {
return fmt.Errorf("remove container %q: %w", s.Name, err)
}
if err = s.startNewContainer(ctx); err != nil {
return err
}
}
slog.Debug("Waiting for corrosion service to be ready.")
if err = WaitReady(ctx, s.DataDir); err != nil {
return err
}
slog.Debug("Corrosion service is ready.")
return nil
}
func (s *DockerService) Stop(ctx context.Context) error {
if err := s.Client.ContainerStop(ctx, s.Name, container.StopOptions{}); err != nil {
return fmt.Errorf("stop container %q: %w", s.Name, err)
}
slog.Debug("Corrosion Docker container stopped.", "name", s.Name)
if err := s.Client.ContainerRemove(ctx, s.Name, container.RemoveOptions{}); err != nil {
return fmt.Errorf("remove container %q: %w", s.Name, err)
}
slog.Debug("Corrosion Docker container removed.", "name", s.Name)
return nil
return s.startNewContainer(ctx)
}
func (s *DockerService) Restart(ctx context.Context) error {
+1 -54
View File
@@ -1,62 +1,9 @@
package corroservice
import (
"context"
"fmt"
"os"
"path/filepath"
"time"
"github.com/BurntSushi/toml"
"github.com/cenkalti/backoff/v4"
"github.com/psviderski/uncloud/internal/corrosion"
)
import "context"
type Service interface {
Start(ctx context.Context) error
Stop(ctx context.Context) error
Restart(ctx context.Context) error
Running() bool
}
// WaitReady waits for the Corrosion service to be ready with the uncloud schema applied.
func WaitReady(ctx context.Context, dataDir string) error {
// Read the config file to get the API address.
configPath := filepath.Join(dataDir, "config.toml")
configData, err := os.ReadFile(configPath)
if err != nil {
return fmt.Errorf("read config file: %w", err)
}
var config Config
if err = toml.Unmarshal(configData, &config); err != nil {
return fmt.Errorf("unmarshal config: %w", err)
}
corro, err := corrosion.NewAPIClient(config.API.Addr)
if err != nil {
return fmt.Errorf("create corrosion API client: %w", err)
}
// Corrosion starts serving the API before applying the schema. Query the cluster table with exponential backoff
// to check if the uncloud schema has been applied.
checkReady := func() error {
rows, err := corro.QueryContext(ctx, "SELECT 1 FROM cluster LIMIT 1")
if err != nil {
return fmt.Errorf("query cluster table: %w", err)
}
defer rows.Close()
return nil
}
b := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(50*time.Millisecond),
backoff.WithMaxInterval(1*time.Second),
backoff.WithMaxElapsedTime(15*time.Second),
)
if err = backoff.Retry(checkReady, backoff.WithContext(b, ctx)); err != nil {
return fmt.Errorf("corrosion service did not become ready: %w", err)
}
return nil
}
+157
View File
@@ -0,0 +1,157 @@
package corroservice
import (
"bufio"
"context"
"fmt"
"log/slog"
"os/exec"
"path/filepath"
"sync"
"syscall"
"time"
)
const (
DefaultCommand = "corrosion"
DefaultDataDir = "/var/lib/uncloud/corrosion"
)
// SubprocessService implements the Service interface by running the service as a subprocess.
type SubprocessService struct {
Command string
DataDir string
cmd *exec.Cmd
running bool
mu sync.Mutex
cancelWatch context.CancelFunc
}
func DefaultSubprocessService() *SubprocessService {
return &SubprocessService{
Command: DefaultCommand,
DataDir: DefaultDataDir,
}
}
// TODO: maybe stop the process if this ctx is cancelled.
func (s *SubprocessService) Start(ctx context.Context) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.running {
return nil
}
return s.startProcess(ctx)
}
func (s *SubprocessService) Restart(ctx context.Context) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.running {
if err := s.stopProcess(); err != nil {
return fmt.Errorf("stop process: %w", err)
}
}
return s.startProcess(ctx)
}
func (s *SubprocessService) Running() bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.running
}
func (s *SubprocessService) startProcess(ctx context.Context) error {
s.cmd = exec.Command(s.Command, "agent", "-c", filepath.Join(s.DataDir, "config.toml"))
// Redirect stdout and stderr to the logger.
stdout, err := s.cmd.StdoutPipe()
if err != nil {
return fmt.Errorf("create stdout pipe: %w", err)
}
stderr, err := s.cmd.StderrPipe()
if err != nil {
return fmt.Errorf("create stderr pipe: %w", err)
}
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
slog.Info("[corrosion]: " + scanner.Text())
}
// TODO: remove
slog.Info("######## corrosion redirect go routine end ########")
}()
go func() {
scanner := bufio.NewScanner(stderr)
for scanner.Scan() {
slog.Error("[corrosion]: " + scanner.Text())
}
}()
if err = s.cmd.Start(); err != nil {
return fmt.Errorf("start process: %w", err)
}
s.running = true
// Watch for process exit to update running status.
go func() {
if err := s.cmd.Wait(); err != nil {
slog.Error("corrosion process exited with error.", "code", s.cmd.ProcessState.ExitCode(), "err", err)
}
s.mu.Lock()
s.running = false
s.mu.Unlock()
}()
// TODO: figure out the waiting process
// Wait for initialization
//timer := time.NewTimer(2 * time.Second)
//defer timer.Stop()
//select {
////case <-timer.C:
//// s.running = true
//// return nil
//case <-watchCtx.Done():
// return fmt.Errorf("process failed to start")
//case <-ctx.Done():
// s.stopProcess()
// return ctx.Err()
//}
return nil
}
func (s *SubprocessService) stopProcess() error {
if s.cmd == nil || s.cmd.Process == nil {
return nil
}
if err := s.cmd.Process.Signal(syscall.SIGTERM); err != nil {
return fmt.Errorf("send SIGTERM: %w", err)
}
// Wait up to 5 seconds for graceful shutdown before killing the process.
done := make(chan error, 1)
go func() {
done <- s.cmd.Wait()
}()
select {
case <-time.After(5 * time.Second):
if err := s.cmd.Process.Kill(); err != nil {
return fmt.Errorf("kill process: %w", err)
}
case err := <-done:
if err != nil {
return fmt.Errorf("process exited with error: %w", err)
}
}
return nil
}
+12 -15
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"log/slog"
"os/exec"
"time"
)
const DefaultSystemdUnit = "uncloud-corrosion.service"
@@ -26,15 +27,6 @@ func (s *SystemdService) Start(ctx context.Context) error {
return s.startOrRestart(ctx, "start")
}
func (s *SystemdService) Stop(_ context.Context) error {
if _, err := exec.Command("systemctl", "stop", s.Unit).Output(); err != nil {
return fmt.Errorf("systemctl stop %s: %w", s.Unit, err)
}
slog.Info("Corrosion systemd service stopped.", "unit", s.Unit)
return nil
}
func (s *SystemdService) Restart(ctx context.Context) error {
return s.startOrRestart(ctx, "restart")
}
@@ -43,15 +35,20 @@ func (s *SystemdService) startOrRestart(ctx context.Context, cmd string) error {
if _, err := exec.Command("systemctl", cmd, s.Unit).Output(); err != nil {
return fmt.Errorf("systemctl %s %s: %w", cmd, s.Unit, err)
}
slog.Debug(fmt.Sprintf("Corrosion systemd service %sed.", cmd), "unit", s.Unit)
slog.Info(fmt.Sprintf("Corrosion systemd service %sed.", cmd), "unit", s.Unit)
slog.Debug("Waiting for corrosion service to be ready.")
if err := WaitReady(ctx, s.DataDir); err != nil {
return err
// Optimistically wait for the corrosion service to start and initialise the database schema before proceeding.
timer := time.NewTimer(2 * time.Second)
defer timer.Stop()
select {
case <-timer.C:
case <-ctx.Done():
return nil
}
slog.Debug("Corrosion service is ready.")
s.running = true
// TODO: run a goroutine to check the status of the service and log any errors in the uncloud log.
s.running = true
return nil
}
+2 -2
View File
@@ -14,13 +14,13 @@ const DBFileName = "machine.db"
func NewDB(path string) (*sqlx.DB, error) {
// Create the database file with 0600 permissions if it doesn't exist, or update permissions if exists.
if _, err := os.Stat(path); os.IsNotExist(err) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, fmt.Errorf("create SQLite database '%s': %w", path, err)
}
file.Close()
} else {
if err = os.Chmod(path, 0o600); err != nil {
if err = os.Chmod(path, 0600); err != nil {
return nil, fmt.Errorf("update SQLite database permissions '%s': %w", path, err)
}
}
+8 -2
View File
@@ -5,10 +5,12 @@ import (
"fmt"
"log/slog"
"net/netip"
"strings"
"sync"
"time"
"github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/pkg/api"
)
// ClusterResolver implements Resolver by tracking containers in the cluster and resolving service names
@@ -82,13 +84,17 @@ func (r *ClusterResolver) updateServiceIPs(containers []store.ContainerRecord) {
continue
}
ctr := record.Container
ctr := api.ServiceContainer{Container: record.Container}
if ctr.ServiceID() == "" || ctr.ServiceName() == "" {
// Container is not part of a service, skip it.
continue
}
newServiceIPs[ctr.ServiceName()] = append(newServiceIPs[ctr.ServiceName()], ip)
// TODO: remove normalisation after implementing service name validation:
//.https://github.com/psviderski/uncloud/issues/53
serviceName := strings.ToLower(ctr.ServiceName())
newServiceIPs[serviceName] = append(newServiceIPs[serviceName], ip)
// Also add the service ID as a valid lookup.
newServiceIPs[ctr.ServiceID()] = append(newServiceIPs[ctr.ServiceID()], ip)
containersCount++
+2 -16
View File
@@ -196,27 +196,13 @@ func (c *Client) RemoveContainer(ctx context.Context, id string, opts container.
return err
}
// PullOptions defines the options for pulling an image from a remote registry.
// This is a copy of image.PullOptions from the Docker API without the PrivilegeFunc field that is non-serialisable.
type PullOptions struct {
All bool
// RegistryAuth is the base64 encoded credentials for the registry.
RegistryAuth string
Platform string
}
type PullImageMessage struct {
Message jsonmessage.JSONMessage
Err error
}
func (c *Client) PullImage(ctx context.Context, image string, opts PullOptions) (<-chan PullImageMessage, error) {
optsBytes, err := json.Marshal(opts)
if err != nil {
return nil, fmt.Errorf("marshal options: %w", err)
}
stream, err := c.grpcClient.PullImage(ctx, &pb.PullImageRequest{Image: image, Options: optsBytes})
func (c *Client) PullImage(ctx context.Context, image string) (<-chan PullImageMessage, error) {
stream, err := c.grpcClient.PullImage(ctx, &pb.PullImageRequest{Image: image})
if err != nil {
return nil, err
}
@@ -1,19 +0,0 @@
//go:build darwin
package docker
import (
"context"
"fmt"
"net/netip"
)
// EnsureUncloudNetwork is a stub for Darwin.
func (c *Controller) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error {
return fmt.Errorf("not supported on Darwin")
}
// Cleanup is a stub for Darwin.
func (c *Controller) Cleanup() error {
return fmt.Errorf("not supported on Darwin")
}
-234
View File
@@ -1,234 +0,0 @@
package docker
import (
"context"
"errors"
"fmt"
"log/slog"
"net/netip"
"strconv"
dockercontainer "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
dnetwork "github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"github.com/docker/docker/libnetwork/iptables"
"github.com/psviderski/uncloud/internal/machine/dns"
"github.com/psviderski/uncloud/internal/machine/firewall"
"github.com/psviderski/uncloud/internal/machine/network"
"github.com/psviderski/uncloud/pkg/api"
)
// EnsureUncloudNetwork creates the Docker bridge network NetworkName with the provided machine subnet
// if it doesn't exist. If the network exists but has a different subnet, it removes and recreates the network.
// It also configures iptables to allow container access from the WireGuard network.
func (c *Controller) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error {
// Ensure the Docker network 'uncloud' is created with the correct subnet.
needsCreation := false
nw, err := c.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{})
if err != nil {
if !client.IsErrNotFound(err) {
return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
}
needsCreation = true
} else if nw.IPAM.Config[0].Subnet != subnet.String() {
// Remove the Docker network if the subnet is different.
// It could be a leftover from a previous incomplete cleanup.
slog.Info(
"Removing Docker network with old subnet.", "name", NetworkName, "subnet", nw.IPAM.Config[0].Subnet,
)
if err = c.client.NetworkRemove(ctx, NetworkName); err != nil {
// It can still fail if the network is in use by a container. Leave it to the user to resolve the issue.
return fmt.Errorf("remove Docker network '%s': %w", NetworkName, err)
}
needsCreation = true
}
if needsCreation {
if _, err = c.client.NetworkCreate(
ctx, NetworkName, dnetwork.CreateOptions{
Driver: "bridge",
Scope: "local",
IPAM: &dnetwork.IPAM{
Config: []dnetwork.IPAMConfig{
{
Subnet: subnet.String(),
},
},
},
Labels: map[string]string{
api.LabelManaged: "",
},
Options: map[string]string{
// Starting with Docker 28.2.0 (https://github.com/moby/moby/pull/49832), we have to explicitly
// allow direct routing from the WireGuard interface to the bridge network.
"com.docker.network.bridge.trusted_host_interfaces": network.WireGuardInterfaceName,
},
},
); err != nil {
return fmt.Errorf("create Docker network '%s': %w", NetworkName, err)
}
slog.Info("Docker network created.", "name", NetworkName, "subnet", subnet.String())
if nw, err = c.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{}); err != nil {
return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
}
}
// Configure iptables to allow WireGuard network to access containers. The Docker daemon should have already
// created the DOCKER-USER chain at this point.
// TODO: check if this works when firewalld used instead of raw iptables. The Docker daemon has a different
// code path for firewalld.
// Bridge name doesn't seem to be documented but this is the source code where it is generated:
// https://github.com/moby/moby/blob/v27.2.1/libnetwork/drivers/bridge/bridge_linux.go#L664
bridgeName := "br-" + nw.ID[:12]
if err = configureIptables(bridgeName, subnet, dnsServer); err != nil {
return fmt.Errorf("configure iptables for Docker network '%s': %w", NetworkName, err)
}
return nil
}
// configureIptables configures iptables rules for the uncloud Docker network.
func configureIptables(bridgeName string, subnet netip.Prefix, dnsServer netip.Addr) error {
ipt := iptables.GetIptable(iptables.IPv4)
// Allow traffic from other machines and their containers through the WG mesh to the Uncloud containers
// on the machine.
wgRule := []string{
"--in-interface", network.WireGuardInterfaceName,
"--out-interface", bridgeName,
"-j", "ACCEPT",
}
if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Insert, wgRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
// Allow DNS queries from Uncloud containers to the embedded DNS server.
for _, proto := range []string{"udp", "tcp"} {
dnsRule := []string{
"--in-interface", bridgeName,
"--dst", dnsServer.String(),
"--protocol", proto,
"--dport", strconv.Itoa(dns.Port),
"-j", "ACCEPT",
}
if err := ipt.ProgramRule(iptables.Filter, firewall.UncloudInputChain, iptables.Insert, dnsRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
}
// Skip masquerading for the container traffic going from the uncloud Docker network through the WG mesh.
// https://uncloud.run/blog/connect-docker-containers-across-hosts-wireguard#step-3-configure-ip-routing
skipMasqueradeRule := []string{
"--src", subnet.String(),
"--out-interface", network.WireGuardInterfaceName,
"-j", "RETURN",
}
// Delete and reinsert the rule to ensure it's at the top of the POSTROUTING chain before the MASQUERADE rule
// added by Docker: POSTROUTING -s 10.210.X.0/24 ! -o br-XXX -j MASQUERADE
if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil {
return fmt.Errorf("delete iptables rule: %w", err)
}
if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Insert, skipMasqueradeRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
return nil
}
// cleanupIptables deletes the iptables rules for the uncloud Docker network.
func cleanupIptables(bridgeName string, subnet netip.Prefix) error {
ipt := iptables.GetIptable(iptables.IPv4)
// Delete the rule allowing traffic from the WireGuard network to the Docker bridge.
wgRule := []string{
"--in-interface", network.WireGuardInterfaceName,
"--out-interface", bridgeName,
"-j", "ACCEPT",
}
if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Delete, wgRule); err != nil {
return fmt.Errorf("delete iptables rule: %w", err)
}
// Delete the rule that skips masquerading for the container traffic going from the uncloud Docker network
// through the WG mesh.
skipMasqueradeRule := []string{
"--src", subnet.String(),
"--out-interface", network.WireGuardInterfaceName,
"-j", "RETURN",
}
if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil {
return fmt.Errorf("delete iptables rule: %w", err)
}
// Rules in uncloud-owned chains will be automatically cleaned up by the machine cleanup.
return nil
}
// Cleanup removes all uncloud-managed containers and the uncloud Docker network.
func (c *Controller) Cleanup() error {
ctx := context.Background()
var errs []error
// Remove uncloud-managed Docker containers.
containers, err := c.client.ContainerList(ctx, dockercontainer.ListOptions{
All: true, // Include stopped containers.
Filters: filters.NewArgs(
filters.Arg("label", api.LabelManaged),
),
})
if err != nil {
errs = append(errs, fmt.Errorf("list uncloud-managed Docker containers: %w", err))
} else if len(containers) > 0 {
slog.Info("Removing uncloud-managed Docker containers.", "count", len(containers))
removed := 0
for _, ctr := range containers {
err = c.client.ContainerStop(ctx, ctr.ID, dockercontainer.StopOptions{})
if err != nil && !client.IsErrNotFound(err) {
errs = append(errs, fmt.Errorf("stop container '%s': %w", ctr.ID, err))
}
err = c.client.ContainerRemove(ctx, ctr.ID, dockercontainer.RemoveOptions{
// Remove anonymous volumes created by the container.
RemoveVolumes: true,
})
if err == nil {
removed++
} else if !client.IsErrNotFound(err) {
errs = append(errs, fmt.Errorf("remove container '%s': %w", ctr.ID, err))
}
}
slog.Info("Removed uncloud-managed Docker containers.", "count", removed)
}
// Remove the uncloud Docker network and related iptables rules.
nw, err := c.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{})
if err == nil {
bridgeName := "br-" + nw.ID[:12]
var subnet netip.Prefix
if len(nw.IPAM.Config) > 0 {
subnet, _ = netip.ParsePrefix(nw.IPAM.Config[0].Subnet)
}
if subnet.IsValid() {
if err = cleanupIptables(bridgeName, subnet); err != nil {
errs = append(errs, fmt.Errorf("cleanup iptables for Docker network '%s': %w", NetworkName, err))
} else {
slog.Info("Cleaned up iptables rules for Docker network.", "name", NetworkName, "bridge", bridgeName)
}
}
if err = c.client.NetworkRemove(ctx, NetworkName); err == nil {
slog.Info("Docker network removed.", "name", NetworkName)
} else if !client.IsErrNotFound(err) {
errs = append(errs, fmt.Errorf("remove Docker network '%s': %w", NetworkName, err))
}
} else if !client.IsErrNotFound(err) {
errs = append(errs, fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err))
}
return errors.Join(errs...)
}
@@ -7,11 +7,12 @@ import (
"log/slog"
"time"
"github.com/docker/docker/api/types/container"
dockercontainer "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/events"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/pkg/api"
)
const (
@@ -23,26 +24,23 @@ const (
SyncInterval = 30 * time.Second
)
// Controller monitors Docker events and synchronises service containers with the cluster store.
type Controller struct {
type Manager struct {
client *client.Client
// machineID is the ID of the machine where the managed Docker daemon is running.
machineID string
client *client.Client
service *Service
store *store.Store
}
func NewController(machineID string, service *Service, store *store.Store) *Controller {
return &Controller{
func NewManager(client *client.Client, machineID string, store *store.Store) *Manager {
return &Manager{
client: client,
machineID: machineID,
client: service.Client,
service: service,
store: store,
}
}
// WaitDaemonReady waits for the Docker daemon to start and be ready to serve requests.
func (c *Controller) WaitDaemonReady(ctx context.Context) error {
func (m *Manager) WaitDaemonReady(ctx context.Context) error {
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
@@ -52,7 +50,7 @@ func (c *Controller) WaitDaemonReady(ctx context.Context) error {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
_, err := c.client.Ping(ctx)
_, err := m.client.Ping(ctx)
if err == nil {
ready = true
break
@@ -69,7 +67,7 @@ func (c *Controller) WaitDaemonReady(ctx context.Context) error {
return nil
}
func (c *Controller) WatchAndSyncContainers(ctx context.Context) error {
func (m *Manager) WatchAndSyncContainers(ctx context.Context) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
// Filter only local container events.
@@ -81,9 +79,9 @@ func (c *Controller) WatchAndSyncContainers(ctx context.Context) error {
}
// Subscribe to Docker events before running the initial sync to avoid missing any events.
eventCh, errCh := c.service.Client.Events(ctx, opts)
eventCh, errCh := m.client.Events(ctx, opts)
slog.Debug("Syncing containers to cluster store before processing Docker events.")
if err := c.syncContainersToStore(ctx); err != nil {
if err := m.syncContainersToStore(ctx); err != nil {
// The deferred cancel will stop the event subscription.
return fmt.Errorf("sync containers to cluster store: %w", err)
}
@@ -128,13 +126,13 @@ func (c *Controller) WatchAndSyncContainers(ctx context.Context) error {
"container_name", e.Actor.Attributes["name"],
"action", e.Action)
if err := c.syncContainersToStore(ctx); err != nil {
if err := m.syncContainersToStore(ctx); err != nil {
return fmt.Errorf("sync containers to cluster store: %w", err)
}
case <-ticker.C:
slog.Debug("Syncing containers to cluster store triggered by a regular interval.",
"interval", SyncInterval)
if err := c.syncContainersToStore(ctx); err != nil {
if err := m.syncContainersToStore(ctx); err != nil {
return fmt.Errorf("sync containers to cluster store: %w", err)
}
case err := <-errCh:
@@ -146,16 +144,32 @@ func (c *Controller) WatchAndSyncContainers(ctx context.Context) error {
}
}
func (c *Controller) syncContainersToStore(ctx context.Context) error {
storeContainers, err := c.store.ListContainers(ctx, store.ListOptions{MachineIDs: []string{c.machineID}})
func (m *Manager) syncContainersToStore(ctx context.Context) error {
storeContainers, err := m.store.ListContainers(ctx, store.ListOptions{MachineIDs: []string{m.machineID}})
if err != nil {
return fmt.Errorf("list containers from store: %w", err)
}
containers, err := c.service.ListServiceContainers(ctx, "", container.ListOptions{})
// List only Uncloud service containers identified by their labels.
containerSummaries, err := m.client.ContainerList(ctx, dockercontainer.ListOptions{
Filters: filters.NewArgs(
filters.Arg("label", api.LabelServiceID),
filters.Arg("label", api.LabelServiceName),
),
})
if err != nil {
// TODO: mark all containers as outdated in the store.
return fmt.Errorf("list service containers: %w", err)
return fmt.Errorf("list Docker containers: %w", err)
}
// Inspect each container to get the full container details.
containers := make([]api.Container, len(containerSummaries))
for i, cs := range containerSummaries {
ctr, err := m.client.ContainerInspect(ctx, cs.ID)
if err != nil {
return fmt.Errorf("inspect container '%s': %w", cs.ID, err)
}
containers[i] = api.Container{ContainerJSON: ctr}
}
// Delete containers from the store that are no longer present in the Docker daemon.
@@ -175,15 +189,15 @@ func (c *Controller) syncContainersToStore(ctx context.Context) error {
var storeErr error
if len(deleteIDs) > 0 {
if err = c.store.DeleteContainers(ctx, store.DeleteOptions{IDs: deleteIDs}); err != nil {
if err = m.store.DeleteContainers(ctx, store.DeleteOptions{IDs: deleteIDs}); err != nil {
storeErr = fmt.Errorf("delete containers from store: %w", err)
}
}
// Create or update the current Docker containers in the store.
for _, ctr := range containers {
if err = c.store.CreateOrUpdateContainer(ctx, ctr, c.machineID); err != nil {
storeErr = errors.Join(storeErr, fmt.Errorf("create or update container '%s': %w", ctr.ID, err))
for _, c := range containers {
if err = m.store.CreateOrUpdateContainer(ctx, c, m.machineID); err != nil {
storeErr = errors.Join(storeErr, fmt.Errorf("create or update container %q: %w", c.ID, err))
}
}
return storeErr
+14
View File
@@ -0,0 +1,14 @@
//go:build darwin
package docker
import (
"context"
"fmt"
"net/netip"
)
// EnsureUncloudNetwork is a stub for Darwin.
func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error {
return fmt.Errorf("not supported on Darwin")
}
+107
View File
@@ -0,0 +1,107 @@
package docker
import (
"context"
"fmt"
"log/slog"
"net/netip"
"strconv"
dnetwork "github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"github.com/docker/docker/libnetwork/iptables"
"github.com/psviderski/uncloud/internal/machine/dns"
"github.com/psviderski/uncloud/internal/machine/firewall"
"github.com/psviderski/uncloud/internal/machine/network"
)
// EnsureUncloudNetwork creates the Docker bridge network NetworkName with the provided machine subnet
// if it doesn't exist. If the network exists but has a different subnet, it removes and recreates the network.
// It also configures iptables to allow container access from the WireGuard network.
func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error {
// Ensure the Docker network 'uncloud' is created with the correct subnet.
needsCreation := false
nw, err := m.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{})
if err != nil {
if !client.IsErrNotFound(err) {
return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
}
needsCreation = true
} else if nw.IPAM.Config[0].Subnet != subnet.String() {
// Remove the Docker network if the subnet is different.
// It could be a leftover from a previous incomplete cleanup.
slog.Info(
"Removing Docker network with old subnet.", "name", NetworkName, "subnet", nw.IPAM.Config[0].Subnet,
)
if err = m.client.NetworkRemove(ctx, NetworkName); err != nil {
// It can still fail if the network is in use by a container. Leave it to the user to resolve the issue.
return fmt.Errorf("remove Docker network '%s': %w", NetworkName, err)
}
needsCreation = true
}
if needsCreation {
if _, err = m.client.NetworkCreate(
ctx, NetworkName, dnetwork.CreateOptions{
Driver: "bridge",
Scope: "local",
IPAM: &dnetwork.IPAM{
Config: []dnetwork.IPAMConfig{
{
Subnet: subnet.String(),
},
},
},
},
); err != nil {
return fmt.Errorf("create Docker network '%s': %w", NetworkName, err)
}
slog.Info("Docker network created.", "name", NetworkName, "subnet", subnet.String())
if nw, err = m.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{}); err != nil {
return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
}
}
// Configure iptables to allow WireGuard network to access containers. The Docker daemon should have already
// created the DOCKER-USER chain at this point.
// TODO: check if this works when firewalld used instead of raw iptables. The Docker daemon has a different
// code path for firewalld.
// Bridge name doesn't seem to be documented but this is the source code where it is generated:
// https://github.com/moby/moby/blob/v27.2.1/libnetwork/drivers/bridge/bridge_linux.go#L664
bridgeName := "br-" + nw.ID[:12]
if err = configureIptables(bridgeName, dnsServer); err != nil {
return fmt.Errorf("configure iptables for Docker network '%s': %w", NetworkName, err)
}
return nil
}
// configureIptables configures iptables rules for the uncloud Docker network.
func configureIptables(bridgeName string, dnsServer netip.Addr) error {
ipt := iptables.GetIptable(iptables.IPv4)
// Allow traffic from other machines and their containers through the WG mesh to the Uncloud containers
// on the machine.
wgRule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"}
if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Insert, wgRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
// Allow DNS queries from Uncloud containers to the embedded DNS server.
for _, proto := range []string{"udp", "tcp"} {
dnsRule := []string{
"--in-interface", bridgeName,
"--dst", dnsServer.String(),
"--protocol", proto,
"--dport", strconv.Itoa(dns.Port),
"-j", "ACCEPT",
}
if err := ipt.ProgramRule(iptables.Filter, firewall.UncloudInputChain, iptables.Insert, dnsRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
}
return nil
}
+53 -70
View File
@@ -2,21 +2,19 @@ package docker
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/netip"
"os"
"regexp"
"slices"
"strconv"
"strings"
"github.com/distribution/reference"
dockercommand "github.com/docker/cli/cli/command"
dockerconfig "github.com/docker/cli/cli/config"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
@@ -48,48 +46,19 @@ var fullDockerIDRegex = regexp.MustCompile(`^[a-f0-9]{64}$`)
type Server struct {
pb.UnimplementedDockerServer
client *client.Client
service *Service
db *sqlx.DB
// internalDNSIP is a function that returns the IP address of the internal DNS server. It may return an empty
// address if the address is unknown (e.g. when the machine is not initialised yet).
internalDNSIP func() netip.Addr
// networkReady is a function that returns true if the Docker network is ready for containers.
networkReady func() bool
// waitForNetworkReady is a function that waits for the Docker network to be ready for containers.
waitForNetworkReady func(ctx context.Context) error
}
// ServerOption configures the Docker server.
type ServerOption func(*Server)
// WithNetworkReady sets the network readiness check function.
func WithNetworkReady(networkReady func() bool) ServerOption {
return func(s *Server) {
s.networkReady = networkReady
}
}
// WithWaitForNetworkReady sets the network readiness wait function.
func WithWaitForNetworkReady(waitForNetworkReady func(ctx context.Context) error) ServerOption {
return func(s *Server) {
s.waitForNetworkReady = waitForNetworkReady
}
}
// NewServer creates a new Docker gRPC server with the provided Docker service.
func NewServer(service *Service, db *sqlx.DB, internalDNSIP func() netip.Addr, opts ...ServerOption) *Server {
s := &Server{
client: service.Client,
service: service,
// NewServer creates a new Docker gRPC server with the provided Docker client.
func NewServer(cli *client.Client, db *sqlx.DB, internalDNSIP func() netip.Addr) *Server {
return &Server{
client: cli,
db: db,
internalDNSIP: internalDNSIP,
}
for _, opt := range opts {
opt(s)
}
return s
}
// CreateContainer creates a new container based on the given configuration.
@@ -149,15 +118,6 @@ func (s *Server) InspectContainer(ctx context.Context, req *pb.InspectContainerR
// StartContainer starts a container with the given ID and options.
func (s *Server) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (*emptypb.Empty, error) {
// Wait for Docker network to be ready before starting the container
if s.waitForNetworkReady != nil {
if err := s.waitForNetworkReady(ctx); err != nil {
return nil, status.Errorf(codes.Unavailable, "Docker network not ready: %v", err)
}
} else if s.networkReady != nil && !s.networkReady() {
return nil, status.Errorf(codes.Unavailable, "Docker network not ready")
}
var opts container.StartOptions
if len(req.Options) > 0 {
if err := json.Unmarshal(req.Options, &opts); err != nil {
@@ -269,6 +229,7 @@ func (s *Server) RemoveContainer(ctx context.Context, req *pb.RemoveContainerReq
func (s *Server) PullImage(req *pb.PullImageRequest, stream grpc.ServerStreamingServer[pb.JSONMessage]) error {
ctx := stream.Context()
// TODO: replace with another JSON serializable type (PullOptions.PrivilegeFunc is not serializable).
var opts image.PullOptions
if len(req.Options) > 0 {
if err := json.Unmarshal(req.Options, &opts); err != nil {
@@ -276,14 +237,6 @@ func (s *Server) PullImage(req *pb.PullImageRequest, stream grpc.ServerStreaming
}
}
if opts.RegistryAuth == "" {
// Try to retrieve the authentication token for the image from the default local Docker config file.
dockerConfig := dockerconfig.LoadDefaultConfigFile(os.Stderr)
if encodedAuth, err := dockercommand.RetrieveAuthTokenFromImage(dockerConfig, req.Image); err == nil {
opts.RegistryAuth = encodedAuth
}
}
respBody, err := s.client.ImagePull(ctx, req.Image, opts)
if err != nil {
return status.Errorf(codes.Internal, err.Error())
@@ -466,7 +419,6 @@ func (s *Server) RemoveVolume(ctx context.Context, req *pb.RemoveVolumeRequest)
}
// CreateServiceContainer creates a new container for the service with the given specifications.
// TODO: move the main logic to the Docker service and remove db dependency from the server.
func (s *Server) CreateServiceContainer(
ctx context.Context, req *pb.CreateServiceContainerRequest,
) (*pb.CreateContainerResponse, error) {
@@ -558,10 +510,10 @@ func (s *Server) CreateServiceContainer(
Memory: spec.Container.Resources.Memory,
MemoryReservation: spec.Container.Resources.MemoryReservation,
},
// Restart service containers if they exit or a machine restarts unless they are explicitly stopped.
// Always restart service containers if they exit or a machine restarts.
// For one-off containers and batch jobs we plan to use a different service type/mode.
RestartPolicy: container.RestartPolicy{
Name: container.RestartPolicyUnlessStopped,
Name: container.RestartPolicyAlways,
},
}
@@ -721,7 +673,7 @@ func (s *Server) verifyDockerVolumesExist(ctx context.Context, mounts []mount.Mo
func (s *Server) InspectServiceContainer(
ctx context.Context, req *pb.InspectContainerRequest,
) (*pb.ServiceContainer, error) {
serviceCtr, err := s.service.InspectServiceContainer(ctx, req.Id)
ctr, err := s.client.ContainerInspect(ctx, req.Id)
if err != nil {
if client.IsErrNotFound(err) {
return nil, status.Errorf(codes.NotFound, err.Error())
@@ -729,14 +681,19 @@ func (s *Server) InspectServiceContainer(
return nil, status.Errorf(codes.Internal, err.Error())
}
ctrBytes, err := json.Marshal(serviceCtr.Container)
ctrBytes, err := json.Marshal(ctr)
if err != nil {
return nil, status.Errorf(codes.Internal, "marshal container: %v", err)
return nil, status.Errorf(codes.Internal, "marshal response: %v", err)
}
specBytes, err := json.Marshal(serviceCtr.ServiceSpec)
var specBytes []byte
err = s.db.QueryRowContext(ctx, `SELECT service_spec FROM containers WHERE id = $1`, ctr.ID).Scan(&specBytes)
if err != nil {
return nil, status.Errorf(codes.Internal, "marshal service spec: %v", err)
if errors.Is(err, sql.ErrNoRows) {
return nil, status.Errorf(codes.NotFound, "service spec not found for container: '%s'", ctr.ID)
}
return nil, status.Errorf(codes.Internal, "get service spec for container '%s' from machine database: %v",
ctr.ID, err)
}
return &pb.ServiceContainer{
@@ -768,28 +725,54 @@ func (s *Server) ListServiceContainers(
return nil, status.Errorf(codes.InvalidArgument, "unmarshal filters: %v", err)
}
opts.Filters = args
} else {
opts.Filters = filters.NewArgs()
}
}
// Only uncloud-managed containers that belong to some service.
opts.Filters.Add("label", api.LabelServiceID)
opts.Filters.Add("label", api.LabelManaged)
containers, err := s.service.ListServiceContainers(ctx, req.ServiceId, opts)
containerSummaries, err := s.client.ContainerList(ctx, opts)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
// Convert to protobuf format.
pbContainers := make([]*pb.ServiceContainer, 0, len(containers))
for _, ctr := range containers {
ctrBytes, err := json.Marshal(ctr.Container)
containers := make([]*pb.ServiceContainer, 0, len(containerSummaries))
for _, cs := range containerSummaries {
if req.ServiceId != "" &&
cs.Labels[api.LabelServiceID] != req.ServiceId && cs.Labels[api.LabelServiceName] != req.ServiceId {
continue
}
ctr, err := s.client.ContainerInspect(ctx, cs.ID)
if err != nil {
if client.IsErrNotFound(err) {
// The listed container may have been removed while we were inspecting other containers.
continue
}
return nil, status.Errorf(codes.Internal, "inspect container %s: %v", cs.ID, err)
}
ctrBytes, err := json.Marshal(ctr)
if err != nil {
return nil, status.Errorf(codes.Internal, "marshal container: %v", err)
}
specBytes, err := json.Marshal(ctr.ServiceSpec)
var specBytes []byte
err = s.db.QueryRowContext(ctx, `SELECT service_spec FROM containers WHERE id = $1`, ctr.ID).Scan(&specBytes)
if err != nil {
return nil, status.Errorf(codes.Internal, "marshal service spec: %v", err)
if errors.Is(err, sql.ErrNoRows) {
// If this happens, there is a bug in the code, or someone manually removed the container from the DB,
// or created a managed container out of band.
slog.Error("Service container not found in machine database.", "id", ctr.ID)
// Just ignore such a container to not fail the list operation as it's not easily recoverable.
continue
}
return nil, status.Errorf(codes.Internal, "get service spec for container '%s' from machine database: %v",
ctr.ID, err)
}
pbContainers = append(pbContainers, &pb.ServiceContainer{
containers = append(containers, &pb.ServiceContainer{
Container: ctrBytes,
ServiceSpec: specBytes,
})
@@ -798,7 +781,7 @@ func (s *Server) ListServiceContainers(
return &pb.ListServiceContainersResponse{
Messages: []*pb.MachineServiceContainers{
{
Containers: pbContainers,
Containers: containers,
},
},
}, nil
-103
View File
@@ -1,103 +0,0 @@
package docker
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"log/slog"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/jmoiron/sqlx"
"github.com/psviderski/uncloud/pkg/api"
)
// Service provides higher-level Docker operations that extends Docker API with Uncloud-specific data
// from the machine database.
type Service struct {
Client *client.Client
db *sqlx.DB
}
// NewService creates a new Docker service instance.
func NewService(client *client.Client, db *sqlx.DB) *Service {
return &Service{
Client: client,
db: db,
}
}
// InspectServiceContainer inspects a Docker container and retrieves its associated ServiceSpec
// from the machine database, returning a complete ServiceContainer.
func (s *Service) InspectServiceContainer(ctx context.Context, nameOrID string) (api.ServiceContainer, error) {
var serviceCtr api.ServiceContainer
ctr, err := s.Client.ContainerInspect(ctx, nameOrID)
if err != nil {
return serviceCtr, err
}
if _, ok := ctr.Config.Labels[api.LabelManaged]; !ok {
return serviceCtr, fmt.Errorf("container '%s' is not managed by Uncloud", nameOrID)
}
serviceCtr.Container = api.Container{ContainerJSON: ctr}
// Retrieve ServiceSpec from the machine database.
var specBytes []byte
err = s.db.QueryRowContext(ctx, `SELECT service_spec FROM containers WHERE id = $1`, ctr.ID).Scan(&specBytes)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
// If this happens, there is a bug in the code, or someone manually removed the container from the DB,
// or created a managed container out of band or by previous uncloud installation.
return serviceCtr, fmt.Errorf("service spec not found for container '%s' in machine DB", ctr.ID)
}
return serviceCtr, fmt.Errorf("get service spec for container '%s' from machine DB: %w", ctr.ID, err)
}
if err = json.Unmarshal(specBytes, &serviceCtr.ServiceSpec); err != nil {
return serviceCtr, fmt.Errorf("unmarshal service spec for container '%s': %w", ctr.ID, err)
}
return serviceCtr, nil
}
// ListServiceContainers lists Docker containers that belong to the service with the given name or ID.
// If serviceIDOrName is empty, all service containers are returned. The opts parameter allows additional filtering.
func (s *Service) ListServiceContainers(
ctx context.Context, serviceNameOrID string, opts container.ListOptions,
) ([]api.ServiceContainer, error) {
if opts.Filters.Len() == 0 {
opts.Filters = filters.NewArgs()
}
// Add labels to existing filters to list only Uncloud-managed service containers.
opts.Filters.Add("label", api.LabelServiceID)
opts.Filters.Add("label", api.LabelManaged)
containerSummaries, err := s.Client.ContainerList(ctx, opts)
if err != nil {
return nil, err
}
var containers []api.ServiceContainer
for _, cs := range containerSummaries {
// Filter by service name or ID if provided.
if serviceNameOrID != "" &&
cs.Labels[api.LabelServiceID] != serviceNameOrID &&
cs.Labels[api.LabelServiceName] != serviceNameOrID {
continue
}
ctr, err := s.InspectServiceContainer(ctx, cs.ID)
if err != nil {
// Log error but continue with other containers.
slog.Error("Failed to inspect service container.", "service", serviceNameOrID, "id", cs.ID, "err", err)
continue
}
containers = append(containers, ctr)
}
return containers, nil
}
@@ -6,8 +6,3 @@ import "fmt"
func ConfigureIptablesChains() error {
return fmt.Errorf("not supported on Darwin")
}
// CleanupIptablesChains is a stub for Darwin.
func CleanupIptablesChains() error {
return fmt.Errorf("not supported on Darwin")
}
+18 -107
View File
@@ -2,13 +2,10 @@ package firewall
import (
"fmt"
"log/slog"
"strconv"
"strings"
"github.com/docker/docker/libnetwork/iptables"
"github.com/psviderski/uncloud/internal/machine/constants"
"github.com/psviderski/uncloud/internal/machine/corroservice"
"github.com/psviderski/uncloud/internal/machine/network"
)
@@ -19,72 +16,25 @@ const (
// ConfigureIptablesChains sets up custom iptables chains and initial firewall rules for Uncloud networking.
func ConfigureIptablesChains() error {
if err := createIptablesChains(); err != nil {
return err
}
ipt4 := iptables.GetIptable(iptables.IPv4)
ipt6 := iptables.GetIptable(iptables.IPv6)
// Allow WireGuard traffic to the machine.
acceptWireGuardRule := []string{"-p", "udp", "--dport", strconv.Itoa(network.WireGuardPort), "-j", "ACCEPT"}
err := ipt4.ProgramRule(iptables.Filter, UncloudInputChain, iptables.Insert, acceptWireGuardRule)
if err != nil {
return fmt.Errorf("insert iptables rule '%s': %w", strings.Join(acceptWireGuardRule, " "), err)
}
// Allow cluster machines to access Machine API via the management IPv6 WireGuard network.
acceptMachineAPIRule := []string{
"-i", network.WireGuardInterfaceName,
"-s", "fdcc::/16",
"-p", "tcp",
"--dport", strconv.Itoa(constants.MachineAPIPort),
"-j", "ACCEPT",
}
// Allow Corrosion gossip traffic from cluster machines via the management IPv6 WireGuard network.
acceptCorrosionGossipRule := []string{
"-i", network.WireGuardInterfaceName,
"-s", "fdcc::/16",
"-p", "udp",
"--dport", strconv.Itoa(corroservice.DefaultGossipPort),
"-j", "ACCEPT",
}
for _, rule := range [][]string{acceptMachineAPIRule, acceptCorrosionGossipRule} {
if err = ipt6.ProgramRule(iptables.Filter, UncloudInputChain, iptables.Insert, rule); err != nil {
return fmt.Errorf("insert ip6tables rule '%s': %w", strings.Join(rule, " "), err)
}
}
return nil
}
// createIptablesChains ensures UNCLOUD-INPUT iptables and ip6tables chains exist and
// there are jump rules from the main INPUT chains.
func createIptablesChains() error {
ipt4 := iptables.GetIptable(iptables.IPv4)
ipt6 := iptables.GetIptable(iptables.IPv6)
for i, ipt := range []*iptables.IPTable{ipt4, ipt6} {
iptBin := "iptables"
if i == 1 {
iptBin = "ip6tables"
}
// Ensure UNCLOUD-INPUT chain exists. All existing rules are flushed.
// Ensure iptables UNCLOUD-INPUT chain with a RETURN rule exists. All existing rules are flushed.
ipt := iptables.GetIptable(iptables.IPv4)
if _, err := ipt.NewChain(UncloudInputChain, iptables.Filter); err != nil {
return fmt.Errorf("create %s chain '%s': %w", iptBin, UncloudInputChain, err)
return fmt.Errorf("create iptables chain '%s': %w", UncloudInputChain, err)
}
if err := ipt.RawCombinedOutput("-t", string(iptables.Filter), "-F", UncloudInputChain); err != nil {
return fmt.Errorf("flush %s chain '%s': %w", iptBin, UncloudInputChain, err)
return fmt.Errorf("flush iptables chain '%s': %w", UncloudInputChain, err)
}
if err := ipt.AddReturnRule(UncloudInputChain); err != nil {
return fmt.Errorf("add the RETURN rule for iptables chain '%s': %w", UncloudInputChain, err)
}
// Ensure the main INPUT chain has a jump rule to the UNCLOUD-INPUT chain before any DROP/REJECT rules.
// Ensure the main iptables INPUT chain has a jump rule to the UNCLOUD-INPUT chain before any DROP/REJECT rules.
jumpRule := []string{"-m", "comment", "--comment", "Uncloud-managed", "-j", UncloudInputChain}
if !ipt.Exists(iptables.Filter, "INPUT", jumpRule...) {
// Look for the first DROP/REJECT rule in the INPUT chain.
out, err := ipt.Raw("-t", string(iptables.Filter), "-L", "INPUT", "--line-numbers")
if err != nil {
return fmt.Errorf("get %s rules for chain '%s': %w", iptBin, UncloudInputChain, err)
return fmt.Errorf("get iptables rules for chain '%s': %w", UncloudInputChain, err)
}
firstRejectRuleNum := 0
@@ -103,60 +53,21 @@ func createIptablesChains() error {
var addJumpRule []string
if firstRejectRuleNum > 0 {
addJumpRule = append([]string{
"-t", string(iptables.Filter),
"-I", "INPUT",
strconv.Itoa(firstRejectRuleNum),
}, jumpRule...)
addJumpRule = append([]string{"-t", string(iptables.Filter), "-I", "INPUT", strconv.Itoa(firstRejectRuleNum)},
jumpRule...)
} else {
addJumpRule = append([]string{
"-t", string(iptables.Filter),
"-A", "INPUT",
}, jumpRule...)
addJumpRule = append([]string{"-t", string(iptables.Filter), "-A", "INPUT"}, jumpRule...)
}
if err = ipt.RawCombinedOutput(addJumpRule...); err != nil {
return fmt.Errorf("add %s rule '%s': %w", iptBin, strings.Join(addJumpRule, " "), err)
return fmt.Errorf("add iptables rule '%s': %w", strings.Join(addJumpRule, " "), err)
}
}
}
return nil
}
// CleanupIptablesChains removes the custom iptables chains and rules created by ConfigureIptablesChains.
func CleanupIptablesChains() error {
ipt4 := iptables.GetIptable(iptables.IPv4)
ipt6 := iptables.GetIptable(iptables.IPv6)
for i, ipt := range []*iptables.IPTable{ipt4, ipt6} {
iptBin := "iptables"
if i == 1 {
iptBin = "ip6tables"
}
// First, remove the jump rule from INPUT chain to UNCLOUD-INPUT.
jumpRule := []string{"-m", "comment", "--comment", "Uncloud-managed", "-j", UncloudInputChain}
if err := ipt.ProgramRule(iptables.Filter, "INPUT", iptables.Delete, jumpRule); err != nil {
return fmt.Errorf("delete %s jump rule from INPUT: %w", iptBin, err)
}
// Flush all rules from UNCLOUD-INPUT chain as it must be empty before deletion.
if err := ipt.RawCombinedOutput("-t", string(iptables.Filter), "-F", UncloudInputChain); err != nil {
// Chain might not exist which is fine.
if !strings.Contains(err.Error(), "No chain") {
return fmt.Errorf("flush %s chain '%s': %w", iptBin, UncloudInputChain, err)
}
}
// Delete the UNCLOUD-INPUT chain.
if err := ipt.RawCombinedOutput("-t", string(iptables.Filter), "-X", UncloudInputChain); err != nil {
// Chain might not exist which is fine.
if !strings.Contains(err.Error(), "No chain") {
return fmt.Errorf("delete %s chain '%s': %w", iptBin, UncloudInputChain, err)
}
} else {
slog.Info(fmt.Sprintf("Deleted %s chain.", iptBin), "chain", UncloudInputChain)
}
// Allow WireGuard traffic to the machine.
acceptWireGuardRule := []string{"-p", "udp", "--dport", strconv.Itoa(network.WireGuardPort), "-j", "ACCEPT"}
err := ipt.ProgramRule(iptables.Filter, UncloudInputChain, iptables.Insert, acceptWireGuardRule)
if err != nil {
return fmt.Errorf("insert iptables rule '%s': %w", strings.Join(acceptWireGuardRule, " "), err)
}
return nil
+89 -154
View File
@@ -13,7 +13,6 @@ import (
"path/filepath"
"slices"
"strconv"
"sync"
"github.com/docker/docker/client"
"github.com/docker/go-connections/sockets"
@@ -24,12 +23,12 @@ import (
apiproxy "github.com/psviderski/uncloud/internal/machine/api/proxy"
"github.com/psviderski/uncloud/internal/machine/caddyconfig"
"github.com/psviderski/uncloud/internal/machine/cluster"
"github.com/psviderski/uncloud/internal/machine/constants"
"github.com/psviderski/uncloud/internal/machine/corroservice"
"github.com/psviderski/uncloud/internal/machine/dns"
machinedocker "github.com/psviderski/uncloud/internal/machine/docker"
"github.com/psviderski/uncloud/internal/machine/network"
"github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/pkg/api"
"github.com/siderolabs/grpc-proxy/proxy"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc"
@@ -61,9 +60,9 @@ type Config struct {
// DockerClient manages system and user containers using the local Docker daemon.
DockerClient *client.Client
// CaddyConfigDir specifies the directory where the machine generates the Caddy reverse proxy configuration file
// for routing external traffic to service containers across the internal network. Default is DataDir/caddy.
CaddyConfigDir string
// CaddyConfigPath specifies where the machine generates the Caddy reverse proxy configuration file for routing
// external traffic to service containers across the internal network. Default is DataDir/caddy/caddy.json.
CaddyConfigPath string
// DNSUpstreams specifies the upstream DNS servers for the embedded internal DNS server.
DNSUpstreams []netip.AddrPort
}
@@ -128,8 +127,8 @@ func (c *Config) SetDefaults() (*Config, error) {
}
}
if cfg.CaddyConfigDir == "" {
cfg.CaddyConfigDir = filepath.Join(cfg.DataDir, "caddy")
if cfg.CaddyConfigPath == "" {
cfg.CaddyConfigPath = filepath.Join(cfg.DataDir, "caddy", "caddy.json")
}
return &cfg, nil
@@ -150,20 +149,11 @@ type Machine struct {
started chan struct{}
// initialised is signalled when the machine is configured as a member of a cluster.
initialised chan struct{}
// networkReady is signalled when the Docker network is configured and ready for containers.
networkReady chan struct{}
// resetting is true when the machine is being reset.
resetting bool
// stop cancels the Run method context to stop the machine gracefully.
stop func()
clusterCtrl *clusterController
// store is the cluster store backed by a distributed Corrosion database.
store *store.Store
cluster *cluster.Cluster
// dockerService provides high-level operations for managing Docker containers.
dockerService *machinedocker.Service
dockerServer *machinedocker.Server
docker *machinedocker.Server
// localMachineServer is the gRPC server for the machine API listening on the local Unix socket.
localMachineServer *grpc.Server
@@ -173,9 +163,6 @@ type Machine struct {
// It proxies requests to the local or remote machine API servers depending on the request targets
// and aggregates responses.
localProxyServer *grpc.Server
// mu protects the Machine from concurrent reads and writes.
mu sync.RWMutex
}
func NewMachine(config *Config) (*Machine, error) {
@@ -223,16 +210,19 @@ func NewMachine(config *Config) (*Machine, error) {
c := cluster.NewCluster(corroStore, corroAdmin)
// Init dependencies for a gRPC Docker server that proxies requests to the local Docker daemon.
dockerCli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return nil, fmt.Errorf("create Docker client: %w", err)
}
dbFilePath := filepath.Join(config.DataDir, DBFileName)
db, err := NewDB(dbFilePath)
if err != nil {
return nil, fmt.Errorf("init machine database: %w", err)
}
dockerService := machinedocker.NewService(config.DockerClient, db)
// Init a local gRPC proxy server that proxies requests to the local or remote machine API servers.
proxyDirector := apiproxy.NewDirector(config.MachineSockPath, constants.MachineAPIPort)
proxyDirector := apiproxy.NewDirector(config.MachineSockPath, APIPort)
localProxyServer := grpc.NewServer(
grpc.ForceServerCodecV2(proxy.Codec()),
grpc.UnknownServiceHandler(
@@ -245,10 +235,8 @@ func NewMachine(config *Config) (*Machine, error) {
state: state,
started: make(chan struct{}),
initialised: make(chan struct{}, 1),
networkReady: make(chan struct{}),
store: corroStore,
cluster: c,
dockerService: dockerService,
localProxyServer: localProxyServer,
proxyDirector: proxyDirector,
}
@@ -257,10 +245,8 @@ func NewMachine(config *Config) (*Machine, error) {
internalDNSIP := func() netip.Addr {
return m.IP()
}
m.dockerServer = machinedocker.NewServer(dockerService, db, internalDNSIP,
machinedocker.WithNetworkReady(m.IsNetworkReady),
machinedocker.WithWaitForNetworkReady(m.WaitForNetworkReady))
m.localMachineServer = newGRPCServer(m, c, m.dockerServer)
m.docker = machinedocker.NewServer(dockerCli, db, internalDNSIP)
m.localMachineServer = newGRPCServer(m, c, m.docker)
if m.Initialised() {
m.initialised <- struct{}{}
@@ -301,9 +287,6 @@ func (m *Machine) IP() netip.Addr {
}
func (m *Machine) Run(ctx context.Context) error {
// Create a cancellable context for the Run method to allow stopping the machine gracefully.
ctx, m.stop = context.WithCancel(ctx)
// Docker dependency is essential for the machine to function. Block until it's ready.
if err := docker.WaitDaemonReady(ctx, m.config.DockerClient); err != nil {
return fmt.Errorf("wait for Docker daemon: %w", err)
@@ -311,7 +294,7 @@ func (m *Machine) Run(ctx context.Context) error {
// Configure and start the corrosion service on the loopback if the machine is not initialised as a cluster
// member. This provides the store required for the machine to initialise a new cluster on it. Once the machine
// is initialised, the corrosion service is managed by the clusterController.
// is initialised, the corrosion service is managed by the networkController.
if !m.Initialised() {
if err := m.configureCorrosion(); err != nil {
return fmt.Errorf("configure corrosion service: %w", err)
@@ -321,7 +304,6 @@ func (m *Machine) Run(ctx context.Context) error {
if err := m.config.CorrosionService.Start(ctx); err != nil {
return fmt.Errorf("start corrosion service: %w", err)
}
slog.Info("Corrosion service started.")
}
// Use an errgroup to coordinate error handling and graceful shutdown of multiple machine components.
@@ -332,49 +314,64 @@ func (m *Machine) Run(ctx context.Context) error {
if err != nil {
return fmt.Errorf("listen machine API unix socket %q: %w", m.config.MachineSockPath, err)
}
errGroup.Go(func() error {
errGroup.Go(
func() error {
slog.Info("Starting local machine API server.", "path", m.config.MachineSockPath)
if err := m.localMachineServer.Serve(machineListener); err != nil {
return fmt.Errorf("local machine API server failed: %w", err)
}
return nil
})
},
)
// Start the local API proxy server.
proxyListener, err := listenUnixSocket(m.config.UncloudSockPath)
if err != nil {
return fmt.Errorf("listen API proxy unix socket %q: %w", m.config.UncloudSockPath, err)
}
errGroup.Go(func() error {
errGroup.Go(
func() error {
slog.Info("Starting local API proxy server.", "path", m.config.UncloudSockPath)
if err := m.localProxyServer.Serve(proxyListener); err != nil {
return fmt.Errorf("local API proxy server failed: %w", err)
}
return nil
})
},
)
// Signal that the machine is ready.
close(m.started)
// Wait for the machine to be initialised as a member of a cluster and run the cluster controller.
errGroup.Go(func() error {
// Control loop for managing components that depend on the machine being initialised as a cluster member.
errGroup.Go(
func() error {
if !m.Initialised() {
slog.Info(
"Waiting for the machine to be initialised as a member of a cluster to start the cluster controller.",
"Waiting for the machine to be initialised as a member of a cluster " +
"to start the network controller.",
)
}
var ctrl *networkController
// Error channel for communicating the termination of the network controller.
errCh := make(chan error)
for {
select {
// Wait for the machine to be initialised as a member of a cluster to start the network controller.
// It can be reset when leaving the cluster and then re-initialised again with a new configuration.
case <-m.initialised:
var err error
m.cluster.UpdateMachineID(m.state.ID)
// Ensure the corrosion config is up to date, including a new gossip address if the machine
// has just joined a cluster.
if err := m.configureCorrosion(); err != nil {
if err = m.configureCorrosion(); err != nil {
return fmt.Errorf("configure corrosion service: %w", err)
}
slog.Info("Configured corrosion service.", "dir", m.config.CorrosionDir)
slog.Info("Starting cluster controller.")
slog.Info("Starting network controller.")
// Update the proxy director's local address to the machine's management IP address, allowing
// the proxy to identify which requests should be proxied to the local machine API server.
m.proxyDirector.UpdateLocalAddress(m.state.Network.ManagementIP.String())
@@ -385,11 +382,11 @@ func (m *Machine) Run(ctx context.Context) error {
),
)
// Create a new caddyconfig controller for managing the Caddy reverse proxy configuration.
// Create a new Caddyfile controller for managing the Caddy reverse proxy configuration.
// It will also serve the current machine ID at /.uncloud-verify to verify Caddy reachability.
caddyconfigCtrl, err := caddyconfig.NewController(m.store, m.config.CaddyConfigDir, m.state.ID)
caddyfileCtrl, err := caddyconfig.NewController(m.store, m.config.CaddyConfigPath, m.state.ID)
if err != nil {
return fmt.Errorf("create caddyconfig controller: %w", err)
return fmt.Errorf("create Caddyfile controller: %w", err)
}
dnsResolver := dns.NewClusterResolver(m.store)
@@ -398,39 +395,49 @@ func (m *Machine) Run(ctx context.Context) error {
return fmt.Errorf("create embedded DNS server: %w", err)
}
m.mu.Lock()
m.clusterCtrl, err = newClusterController(
ctrl, err = newNetworkController(
m.state,
m.store,
proxyServer,
m.config.CorrosionService,
m.dockerService,
m.networkReady,
caddyconfigCtrl,
m.config.DockerClient,
caddyfileCtrl,
dnsServer,
dnsResolver,
)
m.mu.Unlock()
if err != nil {
return fmt.Errorf("initialise cluster controller: %w", err)
return fmt.Errorf("initialise network controller: %w", err)
}
if err = m.clusterCtrl.Run(ctx); err != nil {
return fmt.Errorf("run cluster controller: %w", err)
go func() {
if err = ctrl.Run(ctx); err != nil {
errCh <- fmt.Errorf("run network controller: %w", err)
} else {
slog.Info("Network controller stopped.")
errCh <- nil
}
slog.Info("Cluster controller stopped.")
}()
case err := <-errCh:
if err != nil {
return err
}
ctrl = nil
case <-ctx.Done():
// The context was cancelled before the machine was initialised.
// Wait for the network controller to stop before returning.
if ctrl != nil {
if err := <-errCh; err != nil {
return err
}
}
return nil
})
}
}
},
)
// Shutdown goroutine.
errGroup.Go(func() error {
var err error
errGroup.Go(
func() error {
<-ctx.Done()
slog.Info("Stopping local machine API server.")
// TODO: implement timeout for graceful shutdown.
@@ -444,17 +451,10 @@ func (m *Machine) Run(ctx context.Context) error {
m.proxyDirector.Close()
slog.Info("Local API proxy server stopped.")
// Clean up the machine data and resources if the machine shutdown was initiated by a reset.
if m.resetting {
slog.Info("Cleaning up machine data and resources.")
if err = m.cleanup(); err != nil {
slog.Error("Failed to clean up machine data and resources.", "err", err)
}
}
m.config.DockerClient.Close()
return err
})
return nil
},
)
return errGroup.Wait()
}
@@ -483,7 +483,7 @@ func listenUnixSocket(path string) (net.Listener, error) {
// Ensure the parent directory exists and has the correct group permissions.
parent, _ := filepath.Split(path)
if err = os.MkdirAll(parent, 0o750); err != nil {
if err = os.MkdirAll(parent, 0750); err != nil {
return nil, fmt.Errorf("create directory %q: %w", parent, err)
}
if err = os.Chown(parent, -1, gid); err != nil {
@@ -537,38 +537,15 @@ func (m *Machine) configureCorrosion() error {
return fmt.Errorf("write corrosion config: %w", err)
}
if err := os.WriteFile(schemaPath, []byte(store.Schema), 0o644); err != nil {
if err := os.WriteFile(schemaPath, []byte(store.Schema), 0644); err != nil {
return fmt.Errorf("write corrosion schema: %w", err)
}
return nil
}
// cleanup removes the machine resources and persistent state.
func (m *Machine) cleanup() error {
var errs []error
m.mu.RLock()
clusterCtrl := m.clusterCtrl
m.mu.RUnlock()
if clusterCtrl != nil {
if err := clusterCtrl.Cleanup(); err != nil {
errs = append(errs, fmt.Errorf("cleanup cluster resources: %w", err))
}
}
if err := os.RemoveAll(m.config.DataDir); err != nil {
errs = append(errs,
fmt.Errorf("remove data directory with persistent machine state '%s': %w", m.config.DataDir, err))
} else {
slog.Info("Removed data directory storing persistent machine state.", "path", m.config.DataDir)
}
return errors.Join(errs...)
}
// CheckPrerequisites verifies if the machine meets all necessary system requirements to participate in the cluster.
func (m *Machine) CheckPrerequisites(_ context.Context, _ *emptypb.Empty) (*pb.CheckPrerequisitesResponse, error) {
func (m *Machine) CheckPrerequisites(ctx context.Context, _ *emptypb.Empty) (*pb.CheckPrerequisitesResponse, error) {
// Check DNS port (UDP) availability.
if err := checkDNSPortAvailable(); err != nil {
return &pb.CheckPrerequisitesResponse{
@@ -799,62 +776,20 @@ func (m *Machine) Inspect(_ context.Context, _ *emptypb.Empty) (*pb.MachineInfo,
}, nil
}
// IsNetworkReady returns true if the Docker network is ready for containers.
func (m *Machine) IsNetworkReady() bool {
if !m.Initialised() {
// If machine is not initialized, there's no network to check
return false
}
// Check if network is ready by checking if the networkReady channel has been closed
select {
case <-m.networkReady:
return true
default:
return false
}
}
// WaitForNetworkReady waits for the Docker network to be ready for containers.
// It returns nil when the network is ready or an error if the context is cancelled.
func (m *Machine) WaitForNetworkReady(ctx context.Context) error {
if !m.Initialised() {
// If machine is not initialized, there's no network to wait for
return nil
}
// Wait for network to be ready or context to be cancelled
select {
case <-m.networkReady:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// Reset restores the machine to a clean state, scheduling a graceful shutdown and removing all cluster-related
// configuration and resource. The uncloud daemon will restart the machine if managed by systemd.
func (m *Machine) Reset(_ context.Context, _ *pb.ResetRequest) (*emptypb.Empty, error) {
if !m.Initialised() {
return nil, nil
}
// Check if the machine is already being reset to avoid concurrent resets.
m.mu.Lock()
if m.resetting {
m.mu.Unlock()
return nil, status.Error(codes.FailedPrecondition, "machine is already being reset")
}
m.resetting = true
m.mu.Unlock()
// Reset restores the machine to a clean state, removing all cluster-related сonfiguration and data and scheduling
// a graceful shutdown. The uncloud daemon will restart the machine if managed by systemd.
func (m *Machine) Reset(ctx context.Context, _ *pb.ResetRequest) (*emptypb.Empty, error) {
slog.Info("Resetting machine to a clean state.")
// Trigger the machine shutdown. The resetting boolean informs the machine to clean up its resources on shutdown.
// We can't clean up the resources synchronously here because this is an RPC call that depends on the running
// gRPC server and network.
m.stop()
return &emptypb.Empty{}, nil
// TODO: stop and remove all managed service containers.
// TODO: check if the request is coming from the unix or network socket. For the network socket, the reset should
// be called in a separate goroutine to avoid blocking the RPC response.
// TODO: stop the network controller
// TODO: implement and call Cleanup on the network controller to remove Docker network, WG interface, iptables
// rules, corrosion state, ?stop corrosion service.
// TODO: stop the machine and remove the machine.json state. The daemon should restart it to a clean state.
return &emptypb.Empty{}, status.Error(codes.Unimplemented, "reset machine is not implemented yet")
}
// InspectService returns detailed information about a service and its containers stored in the cluster store.
@@ -889,7 +824,7 @@ func (m *Machine) InspectService(
}
}
ctr := records[0].Container
ctr := api.ServiceContainer{Container: records[0].Container}
svc := &pb.Service{
Id: ctr.ServiceID(),
Name: ctr.ServiceName(),
@@ -12,9 +12,9 @@ import (
"time"
"github.com/cenkalti/backoff/v4"
"github.com/docker/docker/client"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/caddyconfig"
"github.com/psviderski/uncloud/internal/machine/constants"
"github.com/psviderski/uncloud/internal/machine/corroservice"
"github.com/psviderski/uncloud/internal/machine/dns"
"github.com/psviderski/uncloud/internal/machine/docker"
@@ -25,10 +25,11 @@ import (
"google.golang.org/grpc"
)
// clusterController is the main controller for the machine that is a cluster member. It manages components such as
// the WireGuard network, API server listening the WireGuard network, Corrosion service, Docker network and containers,
// and others.
type clusterController struct {
const (
APIPort = 51000
)
type networkController struct {
state *State
store *store.Store
@@ -37,30 +38,26 @@ type clusterController struct {
server *grpc.Server
corroService corroservice.Service
dockerCtrl *docker.Controller
// dockerReady is signalled when Docker is configured and ready for containers.
dockerReady chan<- struct{}
caddyconfigCtrl *caddyconfig.Controller
dockerCli *client.Client
caddyfileCtrl *caddyconfig.Controller
// dnsServer is the embedded internal DNS server for the cluster listening on the machine IP.
dnsServer *dns.Server
dnsResolver *dns.ClusterResolver
// stopped is a channel that is closed when the controller is stopped.
stopped chan struct{}
}
func newClusterController(
func newNetworkController(
state *State,
store *store.Store,
server *grpc.Server,
corroService corroservice.Service,
dockerService *docker.Service,
dockerReady chan<- struct{},
dockerCli *client.Client,
caddyfileCtrl *caddyconfig.Controller,
dnsServer *dns.Server,
dnsResolver *dns.ClusterResolver,
) (*clusterController, error) {
) (
*networkController, error,
) {
slog.Info("Starting WireGuard network.")
wgnet, err := network.NewWireGuardNetwork()
if err != nil {
@@ -68,97 +65,88 @@ func newClusterController(
}
endpointChanges := wgnet.WatchEndpoints()
return &clusterController{
return &networkController{
state: state,
store: store,
wgnet: wgnet,
endpointChanges: endpointChanges,
server: server,
corroService: corroService,
dockerCtrl: docker.NewController(state.ID, dockerService, store),
dockerReady: dockerReady,
caddyconfigCtrl: caddyfileCtrl,
dockerCli: dockerCli,
caddyfileCtrl: caddyfileCtrl,
dnsServer: dnsServer,
dnsResolver: dnsResolver,
stopped: make(chan struct{}),
}, nil
}
func (cc *clusterController) Run(ctx context.Context) error {
defer close(cc.stopped)
func (nc *networkController) Run(ctx context.Context) error {
if err := firewall.ConfigureIptablesChains(); err != nil {
return fmt.Errorf("configure iptables chains: %w", err)
}
if err := cc.ensureDockerNetwork(ctx); err != nil {
return err
}
slog.Info("Docker network configured.")
if err := cc.wgnet.Configure(*cc.state.Network); err != nil {
if err := nc.wgnet.Configure(*nc.state.Network); err != nil {
return fmt.Errorf("configure WireGuard network: %w", err)
}
slog.Info("WireGuard network configured.")
if cc.corroService.Running() {
if nc.corroService.Running() {
// Corrosion service was running before the WireGuard network was configured so we need to restart it.
slog.Info("Restarting corrosion service to apply new configuration with WireGuard network.")
if err := cc.corroService.Restart(ctx); err != nil {
if err := nc.corroService.Restart(ctx); err != nil {
return fmt.Errorf("restart corrosion service: %w", err)
}
slog.Info("Corrosion service restarted.")
} else {
slog.Info("Starting corrosion service.")
if err := cc.corroService.Start(ctx); err != nil {
if err := nc.corroService.Start(ctx); err != nil {
return fmt.Errorf("start corrosion service: %w", err)
}
slog.Info("Corrosion service started.")
}
// TODO: Figure out if we need to manually stop the corrosion service when the context is done or just
// rely on systemd to handle service dependencies on its own.
errGroup, ctx := errgroup.WithContext(ctx)
// Start the network API server. Assume the management IP can't be changed when the network is running.
apiAddr := net.JoinHostPort(cc.state.Network.ManagementIP.String(), strconv.Itoa(constants.MachineAPIPort))
apiAddr := net.JoinHostPort(nc.state.Network.ManagementIP.String(), strconv.Itoa(APIPort))
listener, err := net.Listen("tcp", apiAddr)
if err != nil {
return fmt.Errorf("listen API port: %w", err)
}
errGroup.Go(func() error {
errGroup.Go(
func() error {
slog.Info("Starting network API server.", "addr", apiAddr)
if err := cc.server.Serve(listener); err != nil {
if err := nc.server.Serve(listener); err != nil {
return fmt.Errorf("network API server failed: %w", err)
}
return nil
})
},
)
errGroup.Go(func() error {
slog.Info("Starting embedded DNS resolver.")
if err := cc.dnsResolver.Run(ctx); err != nil {
if err := nc.dnsResolver.Run(ctx); err != nil {
return fmt.Errorf("embedded DNS resolver failed: %w", err)
}
return nil
})
// The Docker network must be created before starting the DNS server because it listens on the machine IP.
errGroup.Go(func() error {
slog.Info("Starting embedded DNS server.")
if err := cc.dnsServer.Run(ctx); err != nil {
if err := nc.dnsServer.Run(ctx); err != nil {
return fmt.Errorf("embedded DNS server failed: %w", err)
}
return nil
})
// Synchronise Docker containers to the cluster store.
// Setup Docker network and synchronise containers to the cluster store.
errGroup.Go(func() error {
slog.Info("Watching Docker containers and syncing them to cluster store.")
return cc.syncDockerContainers(ctx)
return nc.prepareAndWatchDocker(ctx)
})
// Handle machine changes in the cluster. Handling machine and endpoint changes should be done
// in separate goroutines to avoid a deadlock when reconfiguring the network.
errGroup.Go(func() error {
if err := cc.handleMachineChanges(ctx); err != nil {
if err := nc.handleMachineChanges(ctx); err != nil {
return fmt.Errorf("handle new machines: %w", err)
}
return nil
@@ -168,24 +156,24 @@ func (cc *clusterController) Run(ctx context.Context) error {
errGroup.Go(func() error {
for {
select {
case e, ok := <-cc.endpointChanges:
case e, ok := <-nc.endpointChanges:
if !ok {
// The channel was closed, stop watching for changes.
cc.endpointChanges = nil
nc.endpointChanges = nil
return nil
}
cc.state.mu.Lock()
for i := range cc.state.Network.Peers {
if cc.state.Network.Peers[i].PublicKey.Equal(e.PublicKey) {
cc.state.Network.Peers[i].Endpoint = &e.Endpoint
nc.state.mu.Lock()
for i := range nc.state.Network.Peers {
if nc.state.Network.Peers[i].PublicKey.Equal(e.PublicKey) {
nc.state.Network.Peers[i].Endpoint = &e.Endpoint
break
}
}
if err := cc.state.Save(); err != nil {
if err := nc.state.Save(); err != nil {
slog.Error("Failed to save machine state.", "err", err)
}
cc.state.mu.Unlock()
nc.state.mu.Unlock()
slog.Debug("Preserved endpoint change in the machine state.",
"public_key", e.PublicKey, "endpoint", e.Endpoint)
@@ -196,66 +184,48 @@ func (cc *clusterController) Run(ctx context.Context) error {
})
errGroup.Go(func() error {
if err := cc.wgnet.Run(ctx); err != nil {
if err := nc.wgnet.Run(ctx); err != nil {
return fmt.Errorf("WireGuard network failed: %w", err)
}
return nil
})
errGroup.Go(func() error {
slog.Info("Starting caddyconfig controller.")
if err := cc.caddyconfigCtrl.Run(ctx); err != nil {
return fmt.Errorf("caddyconfig controller failed: %w", err)
slog.Info("Starting Caddyconfig controller.")
if err := nc.caddyfileCtrl.Run(ctx); err != nil {
//goland:noinspection GoErrorStringFormat
return fmt.Errorf("Caddyconfig controller failed: %w", err)
}
return nil
})
// Wait for the context to be done and stop the network API server.
errGroup.Go(func() error {
<-ctx.Done()
slog.Info("Stopping network API server.")
// TODO: implement timeout for graceful shutdown.
cc.server.GracefulStop()
nc.server.GracefulStop()
slog.Info("Network API server stopped.")
return nil
})
// Wait for all controllers to finish.
err = errGroup.Wait()
// It's safe to stop the Corrosion service after the controllers depending on it and API server are stopped.
// Use a new context with a timeout as the current context is already canceled.
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if corroErr := cc.corroService.Stop(ctx); corroErr != nil {
err = errors.Join(err, fmt.Errorf("stop corrosion service: %w", corroErr))
} else {
slog.Info("Corrosion service stopped.")
return errGroup.Wait()
}
return err
}
// ensureDockerNetwork ensures that the Docker network is configured and ready for containers.
func (cc *clusterController) ensureDockerNetwork(ctx context.Context) error {
if err := cc.dockerCtrl.WaitDaemonReady(ctx); err != nil {
// prepareAndWatchDocker configures the Docker network and watches local Docker containers to sync them
// to the cluster store.
func (nc *networkController) prepareAndWatchDocker(ctx context.Context) error {
manager := docker.NewManager(nc.dockerCli, nc.state.ID, nc.store)
if err := manager.WaitDaemonReady(ctx); err != nil {
return fmt.Errorf("wait for Docker daemon: %w", err)
}
if err := cc.dockerCtrl.EnsureUncloudNetwork(
ctx,
cc.state.Network.Subnet,
cc.dnsServer.ListenAddr(),
); err != nil {
if err := manager.EnsureUncloudNetwork(ctx, nc.state.Network.Subnet, nc.dnsServer.ListenAddr()); err != nil {
return fmt.Errorf("ensure Docker network: %w", err)
}
slog.Info("Docker network configured.")
// Signal that Docker is ready for containers.
close(cc.dockerReady)
return nil
}
// syncDockerContainers watches local Docker containers and syncs them to the cluster store.
// TODO: move this to the Docker controller.
func (cc *clusterController) syncDockerContainers(ctx context.Context) error {
slog.Info("Watching Docker containers and syncing them to cluster store.")
// Retry to watch and sync containers until the context is done.
boff := backoff.WithContext(backoff.NewExponentialBackOff(
backoff.WithInitialInterval(100*time.Millisecond),
@@ -263,7 +233,7 @@ func (cc *clusterController) syncDockerContainers(ctx context.Context) error {
backoff.WithMaxElapsedTime(0),
), ctx)
watchAndSync := func() error {
if wErr := cc.dockerCtrl.WatchAndSyncContainers(ctx); wErr != nil {
if wErr := manager.WatchAndSyncContainers(ctx); wErr != nil {
slog.Error("Failed to watch and sync containers to cluster store, retrying.", "err", wErr)
return wErr
}
@@ -281,7 +251,7 @@ func (cc *clusterController) syncDockerContainers(ctx context.Context) error {
// handleMachineChanges subscribes to machine changes in the cluster and reconfigures the network peers accordingly
// when changes occur.
func (cc *clusterController) handleMachineChanges(ctx context.Context) error {
func (nc *networkController) handleMachineChanges(ctx context.Context) error {
for {
// Retry to subscribe to machine changes indefinitely until the context is done.
boff := backoff.WithContext(backoff.NewExponentialBackOff(
@@ -296,7 +266,7 @@ func (cc *clusterController) handleMachineChanges(ctx context.Context) error {
err error
)
subscribe := func() error {
if machines, changes, err = cc.store.SubscribeMachines(ctx); err != nil {
if machines, changes, err = nc.store.SubscribeMachines(ctx); err != nil {
slog.Info("Failed to subscribe to machine changes, retrying.", "err", err)
}
return err
@@ -314,7 +284,7 @@ func (cc *clusterController) handleMachineChanges(ctx context.Context) error {
// completes. Skip configuration now and apply it when the store changes are received.
if len(machines) > 0 {
slog.Info("Reconfiguring network peers with the current machines.", "machines", len(machines))
if err = cc.configurePeers(machines); err != nil {
if err = nc.configurePeers(machines); err != nil {
slog.Error("Failed to configure peers.", "err", err)
}
}
@@ -326,11 +296,11 @@ func (cc *clusterController) handleMachineChanges(ctx context.Context) error {
// be reworked as well.
case <-changes:
slog.Info("Cluster machines changed, reconfiguring network peers.")
if machines, err = cc.store.ListMachines(ctx); err != nil {
if machines, err = nc.store.ListMachines(ctx); err != nil {
slog.Error("Failed to list machines.", "err", err)
continue
}
if err = cc.configurePeers(machines); err != nil {
if err = nc.configurePeers(machines); err != nil {
slog.Error("Failed to configure peers.", "err", err)
}
case <-ctx.Done():
@@ -340,23 +310,23 @@ func (cc *clusterController) handleMachineChanges(ctx context.Context) error {
}
}
func (cc *clusterController) configurePeers(machines []*pb.MachineInfo) error {
func (nc *networkController) configurePeers(machines []*pb.MachineInfo) error {
if len(machines) == 0 {
return fmt.Errorf("no machines to configure peers")
}
cc.state.mu.RLock()
currentPeerEndpoints := make(map[string]*netip.AddrPort, len(cc.state.Network.Peers))
for _, p := range cc.state.Network.Peers {
nc.state.mu.RLock()
currentPeerEndpoints := make(map[string]*netip.AddrPort, len(nc.state.Network.Peers))
for _, p := range nc.state.Network.Peers {
currentPeerEndpoints[p.PublicKey.String()] = p.Endpoint
}
cc.state.mu.RUnlock()
nc.state.mu.RUnlock()
// Construct the list of peers from the machine configurations ensuring that the current endpoint is preserved.
peers := make([]network.PeerConfig, 0, len(machines)-1)
for _, m := range machines {
// Skip the current machine.
if m.Id == cc.state.ID {
if m.Id == nc.state.ID {
continue
}
if err := m.Network.Validate(); err != nil {
@@ -389,37 +359,20 @@ func (cc *clusterController) configurePeers(machines []*pb.MachineInfo) error {
}
// Preserve the new list of peers in the machine state.
cc.state.mu.Lock()
cc.state.Network.Peers = peers
err := cc.state.Save()
cc.state.mu.Unlock()
nc.state.mu.Lock()
nc.state.Network.Peers = peers
err := nc.state.Save()
nc.state.mu.Unlock()
if err != nil {
return fmt.Errorf("save machine state: %w", err)
}
cc.state.mu.RLock()
defer cc.state.mu.RUnlock()
if err = cc.wgnet.Configure(*cc.state.Network); err != nil {
nc.state.mu.RLock()
defer nc.state.mu.RUnlock()
if err = nc.wgnet.Configure(*nc.state.Network); err != nil {
return fmt.Errorf("configure network peers: %w", err)
}
return nil
}
// Cleanup cleans up the cluster resources such as the WireGuard network, iptables rules, Docker network and containers.
func (cc *clusterController) Cleanup() error {
// Wait for the controller to stop before cleaning up.
<-cc.stopped
var errs []error
if err := cc.dockerCtrl.Cleanup(); err != nil {
errs = append(errs, fmt.Errorf("cleanup Docker resources: %w", err))
}
if err := cc.wgnet.Cleanup(); err != nil {
errs = append(errs, fmt.Errorf("cleanup WireGuard network: %w", err))
}
if err := firewall.CleanupIptablesChains(); err != nil {
errs = append(errs, fmt.Errorf("cleanup iptables chains: %w", err))
}
return errors.Join(errs...)
}
// TODO: method to shutdown network when leaving a cluster. Regular context cancellation shouldn't bring it down.
+1 -1
View File
@@ -14,7 +14,7 @@ func MachineIP(subnet netip.Prefix) netip.Addr {
}
// ManagementIP returns the IPv6 address of a peer derived from the first 14 bytes of its public key.
// This address always starts with fdcc: and is intended for cluster management traffic.
// This address is intended for cluster management traffic.
func ManagementIP(publicKey secret.Secret) netip.Addr {
bytes := [16]byte{0xfd, 0xcc}
copy(bytes[2:], publicKey[:14])

Some files were not shown because too many files have changed in this diff Show More