Compare commits

..
Author SHA1 Message Date
Pasha Sviderski da09d22b47 docs: draft post wireguard overlay 2025-07-08 21:08:29 +10:00
180 changed files with 20295 additions and 24718 deletions
-1
View File
@@ -1 +0,0 @@
../AI.md
+1 -3
View File
@@ -37,9 +37,7 @@ jobs:
(echo "go.mod or go.sum has changed. Please run 'go mod tidy' and commit the changes." && exit 1) (echo "go.mod or go.sum has changed. Please run 'go mod tidy' and commit the changes." && exit 1)
- name: Run tests - name: Run tests
run: | run: make test
make ucind-image
make test
timeout-minutes: 10 timeout-minutes: 10
check-protobuf: check-protobuf:
-39
View File
@@ -1,39 +0,0 @@
name: Lint
on:
push:
branches:
- "main"
- "test/**"
- "release/**"
pull_request:
branches:
- main
paths:
- "**.go"
- "go.*"
permissions:
contents: read
jobs:
lint:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up Go
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
with:
go-version: "1.23.2"
- name: golangci-lint
uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0
with:
version: v2.2.2
- name: Format code
run: |
make format
git diff --exit-code ||
(echo "Code is not formatted. Please run 'make format' and commit the changes." && exit 1)
timeout-minutes: 10
-42
View File
@@ -1,42 +0,0 @@
version: "2"
run:
concurrency: 4
tests: true
timeout: 5m
linters:
default: none
enable:
- bodyclose
# - dogsled
- dupl
# - errcheck
- gochecknoinits
- goconst
# - gocritic
# - gocyclo
# - godot
# - gosec
# - govet
- ineffassign
- misspell
- nakedret
# - prealloc
# - revive
# - staticcheck
- unconvert
# - unparam
# - unused
- whitespace
exclusions:
rules:
- path: ^test/e2e
linters:
- goconst # constants here add no value, so we skip goconst only for test/e2e.
formatters:
enable:
- gofumpt
- goimports
-8
View File
@@ -6,14 +6,6 @@ backend = "core:go"
"go1.23.10.darwin-arm64.tar.gz" = "sha256:25c64bfa8a8fd8e7f62fb54afa4354af8409a4bb2358c2699a1003b733e6fce5" "go1.23.10.darwin-arm64.tar.gz" = "sha256:25c64bfa8a8fd8e7f62fb54afa4354af8409a4bb2358c2699a1003b733e6fce5"
"go1.23.10.linux-amd64.tar.gz" = "sha256:535f9f81802499f2a7dbfa70abb8fda3793725fcc29460f719815f6e10b5fd60" "go1.23.10.linux-amd64.tar.gz" = "sha256:535f9f81802499f2a7dbfa70abb8fda3793725fcc29460f719815f6e10b5fd60"
[tools.golangci-lint]
version = "2.2.2"
backend = "aqua:golangci/golangci-lint"
[tools.golangci-lint.checksums]
"golangci-lint-2.2.2-darwin-arm64.tar.gz" = "sha256:d84d94d042c0d495fd1746f3d18948a75de163b17a14e8de3ef840928dd2df74"
"golangci-lint-2.2.2-linux-amd64.tar.gz" = "sha256:c27fbde948a87d326feacd21df2f61a9c54dbd2e3bfa185c0a1cd6917a6f964f"
[tools.protoc] [tools.protoc]
version = "27.3" version = "27.3"
backend = "aqua:protocolbuffers/protobuf/protoc" backend = "aqua:protocolbuffers/protobuf/protoc"
-1
View File
@@ -3,7 +3,6 @@ experimental = true
[tools] [tools]
go = "1.23" go = "1.23"
golangci-lint = "2.2.2"
protoc = "27.3" protoc = "27.3"
protoc-gen-go = "1.34.2" protoc-gen-go = "1.34.2"
protoc-gen-go-grpc = "1.5.1" protoc-gen-go-grpc = "1.5.1"
-244
View File
@@ -1,244 +0,0 @@
# AI.md - Uncloud Project Guide
This document provides comprehensive information about the Uncloud project for AI assistants to understand the codebase, architecture, and development practices.
## Project Overview
**Uncloud** is a lightweight clustering and container orchestration tool that enables deployment and management of web applications across cloud VMs and bare metal servers. It creates a secure WireGuard mesh network between Docker hosts and provides automatic service discovery, load balancing, HTTPS ingress, and simple CLI commands for application management.
### Key Characteristics
- **Language**: Go
- **Architecture**: Decentralized, no control plane
- **Target**: Self-hosted infrastructure without Kubernetes complexity
- **License**: View LICENSE file for details
- **Status**: Active development, not yet ready for production
## Core Features
### 🏗️ Infrastructure
- **Multi-machine deployment**: Combine cloud VMs, dedicated servers, and bare metal
- **Zero-config networking**: Automatic WireGuard mesh with NAT traversal
- **Decentralized design**: No central control plane, all machines are equal
- **Service discovery**: Built-in DNS server resolves service names to container IPs
### 🚀 Application Management
- **Docker Compose compatibility**: Uses familiar Docker Compose format
- **Zero-downtime deployments**: Rolling updates without service interruption
- **Automatic HTTPS**: Caddy reverse proxy with Let's Encrypt integration
- **Managed DNS**: Free `*.cluster.uncloud.run` subdomains via Uncloud DNS service
- **Cross-machine scaling**: Run containers across multiple machines
### 🔧 Developer Experience
- **Docker-like CLI**: Familiar commands (`uc` binary)
- **Imperative operations**: Direct commands vs. declarative state reconciliation
- **Remote management**: Control entire infrastructure via SSH to any machine
- **Minimal overhead**: ~150MB RAM footprint per machine
## Architecture
### Core Components
1. **CLI (`uc`)** - Main user interface for cluster management
2. **Daemon (`uncloudd`)** - Machine daemon running on each node
3. **Corrosion** - Distributed SQLite database for cluster state (Fly.io project)
4. **Caddy** - Reverse proxy for HTTPS termination and routing
5. **WireGuard** - Secure mesh networking between machines
### Network Architecture
- Each machine gets unique subnet (e.g., `10.210.0.0/24`, `10.210.1.0/24`)
- Containers get cluster-unique IPs for direct communication
- Automatic peer discovery and key management
- NAT traversal for machines behind firewalls
### State Management
- **CRDT-based distributed storage** using Corrosion
- **Eventually consistent** state across all machines
- **Gossip protocol** (Serf) for state propagation
- **No quorum requirements** - partial network splits remain functional
## Project Structure
### Key Directories
- **`cmd/`**: Contains main applications
- `uncloud/`: CLI tool with subcommands for machine, service, volume management
- `uncloudd/`: Daemon that runs on each machine
- `ucind/`: Development cluster management for testing
- **`internal/`**: Internal implementation packages
- `cli/`: Command-line interface logic
- `machine/`: Machine lifecycle and state management
- `daemon/`: Daemon implementation and gRPC services
- `dns/`: Internal DNS server for service discovery
- **`pkg/`**: Public API packages for external use
- `api/`: Core API types and definitions
- `client/`: Client libraries for interacting with Uncloud
- **`experiment/`**: Experimental features and prototypes
- **`scripts/`**: Installation and utility scripts
- **`test/`**: Test suites and test infrastructure
- **`website/`**: Documentation website (Docusaurus)
- **`misc/`**: Design documents and guides
## Key Technologies
### Core Dependencies
```go
// Networking and orchestration
github.com/docker/docker // Docker API client
github.com/docker/compose/v2 // Docker Compose integration
golang.zx2c4.com/wireguard // WireGuard implementation
github.com/hashicorp/serf // Gossip protocol
// State management
github.com/ipfs/go-ds-crdt // CRDT distributed storage
github.com/dgraph-io/badger/v3 // Embedded database
// Web proxy
github.com/caddyserver/caddy/v2 // HTTP server and reverse proxy
// CLI and UX
github.com/spf13/cobra // CLI framework
github.com/charmbracelet/huh // Interactive forms
// gRPC and networking
google.golang.org/grpc // gRPC framework
github.com/siderolabs/grpc-proxy // gRPC proxy for forwarding
```
## Development Workflow
### Build and Development
```bash
# Build binaries
go build -o uncloud ./cmd/uncloud
go build -o uncloudd ./cmd/uncloudd
```
### Key Make Targets
- `proto`: Generate protobuf code
- `ucind-cluster`: Create development cluster
- `update-dev`: Deploy to development machines
- `demo-reset`: Reset demo environment
- `fmt`: Format code
- `test`: Run all tests
- `lint`: Lint the code using golangci-lint
- `lint-and-fix`: Lint the code and fix issues whenever possible
## CLI Commands Structure
The `uc` CLI provides these main command groups:
### Machine Management
```bash
uc machine init <user@host> # Initialize new cluster
uc machine add <user@host> # Add machine to cluster
uc machine ls # List machines
uc machine rm <name> # Remove machine
```
### Service Management
```bash
uc run <image> # Run container from image
uc deploy # Deploy from compose.yaml
uc scale <service> <count> # Scale service replicas
uc ls # List services
uc rm <service> # Remove service
```
### Context and Connectivity
```bash
uc context ls # List available contexts
uc context use <name> # Switch context
```
### Global Flags
- `--connect`: Connect to remote machine directly, without a config file
- `--uncloud-config`: Override config file path
## Development Guidelines
### Code Organization
- **Package naming**: Use clear, descriptive names
- **Error handling**: Wrap errors with context using `fmt.Errorf`
- **Logging**: Use structured logging with levels
- **gRPC**: Services defined in `internal/machine/api/pb/`
### Testing
- Unit tests alongside source files (`*_test.go`)
- Integration tests in `test/e2e/`
- Test fixtures in `test/fixtures/`
### Dependencies
- Prefer standard library when possible
- Pin versions in `go.mod`
- Document rationale for external dependencies
### Configuration
- Support environment variables for key settings
- Validate configuration early
- Provide sensible defaults
## Troubleshooting and Debugging
### Common Issues
- **Networking**: Check WireGuard status, iptables rules
- **DNS**: Verify service discovery resolution
- **Containers**: Use standard Docker debugging tools
- **State sync**: Check Corrosion logs for replication issues
### Debugging Tools
- Standard Linux networking tools (`ping`, `traceroute`, `wireshark`)
- Docker commands (`docker ps`, `docker logs`)
- SSH access to machines for direct inspection
- gRPC debugging tools
### Logs and Monitoring
- Systemd services (getting logs via `journalctl -u SERVICE_NAME`)
- `uncloud` -- Uncloud daemon
- `uncloud-corrosion` -- Corrosion process
- Machine daemon logs
- Container logs via Docker
## File Patterns and Conventions
### Important Files to Understand
- `cmd/uncloud/main.go`: CLI entry point and command structure
- `internal/cli/cli.go`: CLI implementation and configuration
- `internal/machine/machine.go`: Core machine management
- `pkg/api/`: Public API definitions
- `misc/design.md`: Architecture and design philosophy
- `README.md`: User-facing documentation
### Configuration Files
- `go.mod/go.sum`: Go dependency management
- `Makefile`: Build and development tasks
- `Dockerfile`: Container build instructions forUncloud-in-Docker (used for testing)
This document should help AI assistants understand the project structure, make informed suggestions, and contribute effectively to the Uncloud codebase.
-1
View File
@@ -1 +0,0 @@
./AI.md
+5 -24
View File
@@ -1,5 +1,6 @@
CORROSION_IMAGE ?= ghcr.io/psviderski/corrosion:latest CORROSION_IMAGE ?= ghcr.io/psviderski/corrosion:latest
UCIND_IMAGE ?= ghcr.io/psviderski/ucind:latest UCIND_IMAGE ?= ghcr.io/psviderski/ucind:latest
DOCS_IMAGE ?= ghcr.io/psviderski/uncloud-docs:latest
update-dev: update-dev:
GOOS=linux GOARCH=amd64 go build -o uncloudd-linux-amd64 ./cmd/uncloudd && \ GOOS=linux GOARCH=amd64 go build -o uncloudd-linux-amd64 ./cmd/uncloudd && \
@@ -45,11 +46,6 @@ proto:
protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative \ protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative \
--proto_path=. --proto_path=internal/machine/api/vendor internal/machine/api/pb/*.proto --proto_path=. --proto_path=internal/machine/api/vendor internal/machine/api/pb/*.proto
.PHONY: proto-mise
proto-mise:
mise exec -- protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative \
--proto_path=. --proto_path=internal/machine/api/vendor internal/machine/api/pb/*.proto
.PHONY: corrosion-image .PHONY: corrosion-image
corrosion-image: corrosion-image:
docker build -t "$(CORROSION_IMAGE)" --target corrosion . docker build -t "$(CORROSION_IMAGE)" --target corrosion .
@@ -71,13 +67,9 @@ test:
ifeq ($(TEST_NAME),) ifeq ($(TEST_NAME),)
go test -count=1 -v ./... go test -count=1 -v ./...
else else
go test -race -count=1 -v -run ^$(TEST_NAME)$$ ./... go test -count=1 -v -run ^$(TEST_NAME)$$ ./...
endif endif
.PHONY: test-e2e
test-e2e:
go test -race -count=1 -v ./test/e2e
.PHONY: test-clean .PHONY: test-clean
test-clean: test-clean:
@CONTAINERS=$$(docker ps --filter "name=ucind-test" -q); \ @CONTAINERS=$$(docker ps --filter "name=ucind-test" -q); \
@@ -97,17 +89,6 @@ test-clean:
vet: vet:
go vet ./... go vet ./...
.PHONY: format fmt .PHONY: docs-image-push
format fmt: docs-image:
GOOS=linux golangci-lint fmt docker buildx build --push --platform linux/amd64,linux/arm64 -t "$(DOCS_IMAGE)" ./docs
LINT_TARGETS := lint lint-and-fix
.PHONY: $(LINT_TARGETS) _lint
$(LINT_TARGETS): _lint
lint: ARGS=
lint-and-fix: ARGS=--fix
_lint:
# Explicitly set OS to Linux to not skip *_linux.go files when running on macOS.
# Uncloud daemon won't likely support OS other than Linux anytime soon, so for now we can rely on that.
GOOS=linux golangci-lint run $(ARGS)
+10 -19
View File
@@ -1,10 +1,10 @@
<div align="center"> <div align="center">
<img src="./website/landing/images/logo.svg" height="100" width="100" alt="Uncloud logo"/> <img src="./website/images/logo.svg" height="100" width="100" alt="Uncloud logo"/>
<h1>Uncloud</h1> <h1>Uncloud</h1>
<p><strong>Docker simplicity. Multi-machine power.</strong></p> <p><strong>Docker simplicity. Multi-machine power.</strong></p>
<p> <p>
<a href="https://uncloud.run/docs"><img src="https://img.shields.io/badge/Docs-blue.svg?style=for-the-badge&logo=gitbook&logoColor=white" alt="Documentation"></a> <a href="https://docs.uncloud.run"><img src="https://img.shields.io/badge/Docs-blue.svg?style=for-the-badge&logo=gitbook&logoColor=white" alt="Documentation"></a>
<a href="https://discord.gg/eR35KQJhPu"><img src="https://img.shields.io/badge/discord-5865F2.svg?style=for-the-badge&logo=discord&logoColor=white" alt="Join Discord"></a> <a href="https://discord.gg/eR35KQJhPu"><img src="https://img.shields.io/badge/discord-5865F2.svg?style=for-the-badge&logo=discord&logoColor=white" alt="Join Discord"></a>
<a href="https://x.com/psviderski"><img src="https://img.shields.io/badge/follow-black?style=for-the-badge&logo=X&logoColor=while" alt="Follow on X"></a> <a href="https://x.com/psviderski"><img src="https://img.shields.io/badge/follow-black?style=for-the-badge&logo=X&logoColor=while" alt="Follow on X"></a>
<a href="https://github.com/sponsors/psviderski"><img src="https://img.shields.io/badge/Donate-EA4AAA.svg?style=for-the-badge&logo=githubsponsors&logoColor=white" alt="Donate"></a> <a href="https://github.com/sponsors/psviderski"><img src="https://img.shields.io/badge/Donate-EA4AAA.svg?style=for-the-badge&logo=githubsponsors&logoColor=white" alt="Donate"></a>
@@ -52,11 +52,11 @@ complexity of Kubernetes.
## 🎬 Quick demo ## 🎬 Quick demo
The screenshot below demonstrates how I use Uncloud to deploy https://uncloud.run website to 2 remote machines from The screenshot below demonstrates how I use Uncloud to deploy the [Uncloud Documentation](https://docs.uncloud.run)
the [`compose.yaml`](website/compose.yaml) file on my local machine. website to 2 remote machines (why not?) from the [`compose.yaml`](docs/compose.yaml) file on my local machine.
It exposes the container port `8000/tcp` as HTTPS on the domain `uncloud.run`, served by the Caddy reverse proxy on the It exposes the container port `8000/tcp` as HTTPS on the domain `docs.uncloud.run`, served by the Caddy reverse proxy on
remote machines. All managed by Uncloud. the remote machines. All managed by Uncloud.
![Uncloud compose deployment demo](.github/images/compose-deploy.jpg) ![Uncloud compose deployment demo](.github/images/compose-deploy.jpg)
@@ -97,7 +97,7 @@ platform, whether you're running on a $5 VPS, a spare Mac mini, or a rack of bar
curl -fsS https://get.uncloud.run/install.sh | sh curl -fsS https://get.uncloud.run/install.sh | sh
``` ```
See [Installation](https://uncloud.run/docs/getting-started/install-cli) for more options. See [Installation](https://docs.uncloud.run/getting-started/install-cli) for more options.
2. Initialise your first machine: 2. Initialise your first machine:
@@ -130,15 +130,15 @@ platform, whether you're running on a $5 VPS, a spare Mac mini, or a rack of bar
uncloud-uninstall uncloud-uninstall
``` ```
View the [Documentation](https://uncloud.run/docs) for more information. View the [Documentation](https://docs.uncloud.run) for more information.
## ⚙️ How it works ## ⚙️ How it works
Check out the [design document](misc/design.md) to understand Uncloud's design philosophy and goals. Check out the [design document](docs/design.md) to understand Uncloud's design philosophy and goals.
Here is a diagram of an Uncloud multi-provider cluster of 3 machines: Here is a diagram of an Uncloud multi-provider cluster of 3 machines:
![Diagram: multi-provider cluster of 3 machines](website/landing/images/diagram.webp) ![Diagram: multi-provider cluster of 3 machines](website/images/diagram.webp)
<details> <details>
<summary>Peek under the hood to see what happens when you run certain commands.</summary> <summary>Peek under the hood to see what happens when you run certain commands.</summary>
@@ -323,15 +323,6 @@ SQLite database used to share Uncloud's cluster state.
features, and be the first to know when it's ready for production use. features, and be the first to know when it's ready for production use.
* Watch this repository for releases. * Watch this repository for releases.
## 💖 Sponsors
These companies and projects are helping Uncloud with their generous sponsorship and/or services:
<!-- Sentry -->
<a href="https://sentry.io/welcome/">
<img height="100" alt="Sentry" src="https://github.com/user-attachments/assets/6c1439c0-d20d-40dc-a669-c9aa94651dfa" />
</a>
## ❤️ Contributors ## ❤️ Contributors
Thank you [@cedws](https://github.com/cedws) for being the first contributor to Uncloud! 🎉 Thank you [@cedws](https://github.com/cedws) for being the first contributor to Uncloud! 🎉
+1 -2
View File
@@ -2,9 +2,8 @@ package cluster
import ( import (
"fmt" "fmt"
"github.com/psviderski/uncloud/internal/ucind"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/psviderski/uncloud/internal/ucind"
) )
func NewCreateCommand() *cobra.Command { func NewCreateCommand() *cobra.Command {
+1 -2
View File
@@ -2,9 +2,8 @@ package cluster
import ( import (
"fmt" "fmt"
"github.com/psviderski/uncloud/internal/ucind"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/psviderski/uncloud/internal/ucind"
) )
func NewRemoveCommand() *cobra.Command { func NewRemoveCommand() *cobra.Command {
+1 -8
View File
@@ -21,7 +21,6 @@ type deployOptions struct {
profiles []string profiles []string
services []string services []string
noBuild bool noBuild bool
recreate bool
context string context string
} }
@@ -51,8 +50,6 @@ func NewDeployCommand() *cobra.Command {
"Name of the cluster context to deploy to (default is the current context)") "Name of the cluster context to deploy to (default is the current context)")
cmd.Flags().BoolVarP(&opts.noBuild, "no-build", "n", false, cmd.Flags().BoolVarP(&opts.noBuild, "no-build", "n", false,
"Do not build images before deploying services. (default false)") "Do not build images before deploying services. (default false)")
cmd.Flags().BoolVar(&opts.recreate, "recreate", false,
"Recreate containers even if their configuration and image haven't changed.")
// TODO: Consider adding a filter flag to specify which machines to deploy to but keep the rest running. // TODO: Consider adding a filter flag to specify which machines to deploy to but keep the rest running.
// Could be useful to test a new version on a subset of machines before rolling out to all. // Could be useful to test a new version on a subset of machines before rolling out to all.
@@ -111,11 +108,7 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
} }
defer clusterClient.Close() defer clusterClient.Close()
var strategy deploy.Strategy composeDeploy, err := compose.NewDeployment(ctx, clusterClient, project)
if opts.recreate {
strategy = &deploy.RollingStrategy{ForceRecreate: true}
}
composeDeploy, err := compose.NewDeploymentWithStrategy(ctx, clusterClient, project, strategy)
if err != nil { if err != nil {
return fmt.Errorf("create compose deployment: %w", err) return fmt.Errorf("create compose deployment: %w", err)
} }
+6 -7
View File
@@ -41,7 +41,7 @@ func NewAddCommand() *cobra.Command {
if err != nil { if err != nil {
return fmt.Errorf("parse remote machine: %w", err) return fmt.Errorf("parse remote machine: %w", err)
} }
remoteMachine := &cli.RemoteMachine{ remoteMachine := cli.RemoteMachine{
User: user, User: user,
Host: host, Host: host,
Port: port, Port: port,
@@ -59,12 +59,11 @@ func NewAddCommand() *cobra.Command {
cmd.Flags().StringVar( cmd.Flags().StringVar(
&opts.publicIP, "public-ip", "auto", &opts.publicIP, "public-ip", "auto",
"Public IP address of the machine for ingress configuration. Use 'auto' for automatic detection, "+ "Public IP address of the machine for ingress configuration. Use 'auto' for automatic detection, "+
fmt.Sprintf("blank '' or '%s' to disable ingress on this machine, or specify an IP address.", PublicIPNone), "blank '' or 'none' to disable ingress on this machine, or specify an IP address.",
) )
cmd.Flags().StringVarP( cmd.Flags().StringVarP(
&opts.sshKey, "ssh-key", "i", "", &opts.sshKey, "ssh-key", "i", "~/.ssh/id_ed25519",
fmt.Sprintf("Path to SSH private key for remote login (if not already added to SSH agent). (default %q)", "Path to SSH private key for remote login (if not already added to SSH agent).",
cli.DefaultSSHKeyPath),
) )
cmd.Flags().StringVar( cmd.Flags().StringVar(
&opts.version, "version", "latest", &opts.version, "version", "latest",
@@ -78,12 +77,12 @@ func NewAddCommand() *cobra.Command {
return cmd return cmd
} }
func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine, opts addOptions) error { func add(ctx context.Context, uncli *cli.CLI, remoteMachine cli.RemoteMachine, opts addOptions) error {
var publicIP *netip.Addr var publicIP *netip.Addr
switch opts.publicIP { switch opts.publicIP {
case "auto": case "auto":
publicIP = &netip.Addr{} publicIP = &netip.Addr{}
case "", PublicIPNone: case "", "none":
publicIP = nil publicIP = nil
default: default:
ip, err := netip.ParseAddr(opts.publicIP) ip, err := netip.ParseAddr(opts.publicIP)
-6
View File
@@ -1,6 +0,0 @@
package machine
const (
// PublicIPNone is the value used to indicate removal of public IP
PublicIPNone = "none"
)
+4 -5
View File
@@ -77,12 +77,11 @@ func NewInitCommand() *cobra.Command {
cmd.Flags().StringVar( cmd.Flags().StringVar(
&opts.publicIP, "public-ip", "auto", &opts.publicIP, "public-ip", "auto",
"Public IP address of the machine for ingress configuration. Use 'auto' for automatic detection, "+ "Public IP address of the machine for ingress configuration. Use 'auto' for automatic detection, "+
fmt.Sprintf("blank '' or '%s' to disable ingress on this machine, or specify an IP address.", PublicIPNone), "blank '' or 'none' to disable ingress on this machine, or specify an IP address.",
) )
cmd.Flags().StringVarP( cmd.Flags().StringVarP(
&opts.sshKey, "ssh-key", "i", "", &opts.sshKey, "ssh-key", "i", "~/.ssh/id_ed25519",
fmt.Sprintf("Path to SSH private key for remote login (if not already added to SSH agent). (default %q)", "Path to SSH private key for remote login (if not already added to SSH agent).",
cli.DefaultSSHKeyPath),
) )
cmd.Flags().StringVar( cmd.Flags().StringVar(
&opts.version, "version", "latest", &opts.version, "version", "latest",
@@ -106,7 +105,7 @@ func initCluster(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteM
switch opts.publicIP { switch opts.publicIP {
case "auto": case "auto":
publicIP = &netip.Addr{} publicIP = &netip.Addr{}
case "", PublicIPNone: case "", "none":
publicIP = nil publicIP = nil
default: default:
ip, err := netip.ParseAddr(opts.publicIP) ip, err := netip.ParseAddr(opts.publicIP)
-47
View File
@@ -1,47 +0,0 @@
package machine
import (
"context"
"fmt"
"github.com/psviderski/uncloud/internal/cli"
"github.com/spf13/cobra"
)
func NewRenameCommand() *cobra.Command {
var contextName string
cmd := &cobra.Command{
Use: "rename OLD_NAME NEW_NAME",
Short: "Rename a machine in the cluster.",
Long: `Rename a machine in the cluster.
This command changes the name of an existing machine while preserving all other
configuration including network settings, public IP, and cluster membership.`,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
uncli := cmd.Context().Value("cli").(*cli.CLI)
return rename(cmd.Context(), uncli, contextName, args[0], args[1])
},
}
cmd.Flags().StringVarP(
&contextName, "context", "c", "",
"Name of the cluster context. (default is the current context)",
)
return cmd
}
func rename(ctx context.Context, uncli *cli.CLI, contextName, oldName, newName string) error {
client, err := uncli.ConnectCluster(ctx, contextName)
if err != nil {
return err
}
defer client.Close()
machine, err := client.RenameMachine(ctx, oldName, newName)
if err != nil {
return fmt.Errorf("rename machine: %w", err)
}
fmt.Printf("Machine %q renamed to %q (ID: %s)\n", oldName, machine.Name, machine.Id)
return nil
}
+21 -70
View File
@@ -14,13 +14,12 @@ import (
"github.com/docker/compose/v2/pkg/progress" "github.com/docker/compose/v2/pkg/progress"
"github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/container"
"github.com/psviderski/uncloud/internal/cli" "github.com/psviderski/uncloud/internal/cli"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/pkg/api" "github.com/psviderski/uncloud/pkg/api"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
type removeOptions struct { type removeOptions struct {
noReset bool force bool
yes bool yes bool
context string context string
} }
@@ -31,7 +30,7 @@ func NewRmCommand() *cobra.Command {
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "rm MACHINE", Use: "rm MACHINE",
Aliases: []string{"remove", "delete"}, Aliases: []string{"remove", "delete"},
Short: "Remove a machine from a cluster and reset it.", Short: "Remove a machine from a cluster.",
Args: cobra.ExactArgs(1), Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
uncli := cmd.Context().Value("cli").(*cli.CLI) uncli := cmd.Context().Value("cli").(*cli.CLI)
@@ -43,14 +42,11 @@ func NewRmCommand() *cobra.Command {
"Name of the cluster context. (default is the current context)") "Name of the cluster context. (default is the current context)")
cmd.Flags().BoolVarP(&opts.yes, "yes", "y", false, cmd.Flags().BoolVarP(&opts.yes, "yes", "y", false,
"Do not prompt for confirmation before removing the machine.") "Do not prompt for confirmation before removing the machine.")
cmd.Flags().BoolVar(&opts.noReset, "no-reset", false,
"Do not reset the machine after removing it from the cluster. This will leave all containers and data intact.")
return cmd return cmd
} }
func remove(ctx context.Context, uncli *cli.CLI, nameOrID string, opts removeOptions) error { func remove(ctx context.Context, uncli *cli.CLI, machineName string, opts removeOptions) error {
// TODO: automatically choose a connection to the machine that is not being removed.
client, err := uncli.ConnectCluster(ctx, opts.context) client, err := uncli.ConnectCluster(ctx, opts.context)
if err != nil { if err != nil {
return fmt.Errorf("connect to cluster: %w", err) return fmt.Errorf("connect to cluster: %w", err)
@@ -58,46 +54,22 @@ func remove(ctx context.Context, uncli *cli.CLI, nameOrID string, opts removeOpt
defer client.Close() defer client.Close()
// Verify the machine exists and list all service containers on it including stopped ones. // Verify the machine exists and list all service containers on it including stopped ones.
mctx, machines, err := api.ProxyMachinesContext(ctx, client, []string{nameOrID}) listCtx, machines, err := api.ProxyMachinesContext(ctx, client, []string{machineName})
if err != nil { if err != nil {
return err return err
} }
if len(machines) == 0 { if len(machines) == 0 {
return fmt.Errorf("machine '%s' not found in the cluster", nameOrID) return fmt.Errorf("machine '%s' not found in the cluster", machineName)
} }
m := machines[0].Machine m := machines[0].Machine
// Verify if the machine being removed is the proxy machine we're connected to.
proxyMachine, err := client.MachineClient.Inspect(ctx, nil)
if err != nil {
return fmt.Errorf("inspect proxy machine: %w", err)
}
if proxyMachine.Id == m.Id {
allMachines, err := client.ListMachines(ctx, nil)
if err != nil {
return fmt.Errorf("list machines: %w", err)
}
if len(allMachines) > 1 {
return errors.New("cannot remove the machine you are currently connected to. " +
"Please connect to another machine in the cluster and try again. " +
"Use --connect flag or update 'connections' for the cluster context in your Uncloud config")
// It's ok to remove the proxy machine if it's the last one in the cluster.
}
}
// TODO: mark the machine as being removed and unschedulable when this is possible to prevent new containers
// from being scheduled on it while the removal is in progress.
reset := !opts.noReset
var containers []api.ServiceContainer
reachable := false
if reset {
// Check if the machine is up and has service containers.
listOpts := container.ListOptions{All: true} listOpts := container.ListOptions{All: true}
machineContainers, err := client.Docker.ListServiceContainers(mctx, "", listOpts) machineContainers, err := client.Docker.ListServiceContainers(listCtx, "", listOpts)
if err == nil { if err != nil {
reachable = true return fmt.Errorf("list containers: %w", err)
containers = machineContainers[0].Containers }
containers := machineContainers[0].Containers
if len(containers) > 0 { if len(containers) > 0 {
plural := "" plural := ""
if len(containers) > 1 { if len(containers) > 1 {
@@ -106,18 +78,11 @@ func remove(ctx context.Context, uncli *cli.CLI, nameOrID string, opts removeOpt
fmt.Printf("Found %d service container%s on machine '%s':\n", len(containers), plural, m.Name) fmt.Printf("Found %d service container%s on machine '%s':\n", len(containers), plural, m.Name)
fmt.Println(formatContainerTree(containers)) fmt.Println(formatContainerTree(containers))
fmt.Println() fmt.Println()
fmt.Println("This will remove all service containers from the machine, remove it from the cluster, " + fmt.Println("This will remove all service containers on the machine, reset it to the uninitialised state, " +
"and reset it to the uninitialised state.") "and remove it from the cluster.")
} else { } else {
fmt.Printf("No service containers found on machine '%s'.\n", m.Name) fmt.Printf("No service containers found on machine '%s'.\n", m.Name)
fmt.Println("This will remove the machine from the cluster and reset it to the uninitialised state.") fmt.Println("This will reset the machine to the uninitialised state and remove it from the cluster.")
}
} else {
fmt.Printf("This will remove machine '%s' from the cluster without resetting it as it's unreachable.\n",
m.Name)
}
} else {
fmt.Printf("This will remove machine '%s' from the cluster without resetting it.\n", m.Name)
} }
if !opts.yes { if !opts.yes {
@@ -131,37 +96,23 @@ func remove(ctx context.Context, uncli *cli.CLI, nameOrID string, opts removeOpt
} }
} }
if reset && len(containers) > 0 { if len(containers) > 0 {
err = progress.RunWithTitle(ctx, func(ctx context.Context) error { err = progress.RunWithTitle(ctx, func(ctx context.Context) error {
return removeContainers(ctx, client, containers) return removeContainers(ctx, client, containers)
}, uncli.ProgressOut(), "Removing containers") }, uncli.ProgressOut(), "Removing containers")
if err != nil { if err != nil {
return fmt.Errorf("remove containers: %w", err) return fmt.Errorf("remove containers: %w", err)
} }
fmt.Println() fmt.Println()
} }
if _, err = client.RemoveMachine(ctx, &pb.RemoveMachineRequest{Id: m.Id}); err != nil { // TODO: 4. Implement and call Reset via Machine API to reset the machine state to uninitialised.
return fmt.Errorf("remove machine from cluster: %w", err) // TODO: 5. Remove the machine from the cluster store.
}
fmt.Printf("Machine '%s' removed from the cluster.\n", m.Name)
if reset && reachable { return fmt.Errorf("resetting machine is not fully implemented yet")
_, err = client.MachineClient.Reset(mctx, &pb.ResetRequest{}) //fmt.Printf("Machine '%s' removed from the cluster.\n", m.Name)
if err != nil { //return nil
fmt.Printf("WARNING: Failed to reset machine: %v\n", err)
} else {
fmt.Println("Machine reset initiated and will complete in the background.")
}
}
// TODO: remove the connection to the machine from the uncloud config if it exists. We need a way to associate
// the machine with its connection in the config, e.g. by storing the machine name in the connection metadata.
// TODO: If Caddy was running on this machine and a cluster domain is reserved,
// let the user know that the DNS records should be updated.
return nil
} }
// formatContainerTree formats a list of containers grouped by service as a tree structure. // formatContainerTree formats a list of containers grouped by service as a tree structure.
-2
View File
@@ -14,9 +14,7 @@ func NewRootCommand() *cobra.Command {
NewAddCommand(), NewAddCommand(),
NewInitCommand(), NewInitCommand(),
NewListCommand(), NewListCommand(),
NewRenameCommand(),
NewRmCommand(), NewRmCommand(),
NewUpdateCommand(),
NewTokenCommand(), NewTokenCommand(),
) )
return cmd return cmd
+1 -2
View File
@@ -2,10 +2,9 @@ package machine
import ( import (
"fmt" "fmt"
"github.com/spf13/cobra"
"github.com/psviderski/uncloud/internal/daemon" "github.com/psviderski/uncloud/internal/daemon"
"github.com/psviderski/uncloud/internal/machine" "github.com/psviderski/uncloud/internal/machine"
"github.com/spf13/cobra"
) )
type tokenOptions struct { type tokenOptions struct {
-128
View File
@@ -1,128 +0,0 @@
package machine
import (
"context"
"fmt"
"net/netip"
"github.com/psviderski/uncloud/internal/cli"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/spf13/cobra"
)
type updateOptions struct {
name string
publicIP string
context string
}
func NewUpdateCommand() *cobra.Command {
opts := updateOptions{}
cmd := &cobra.Command{
Use: "update",
Short: "Update machine configuration in the cluster.",
Long: `Update machine configuration in the cluster.
This command allows setting various machine properties including:
- Machine name (--name)
- Public IP address (--public-ip)
At least one flag must be specified to perform an update operation.`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
uncli := cmd.Context().Value("cli").(*cli.CLI)
return update(cmd.Context(), uncli, cmd, opts, args[0])
},
}
cmd.Flags().StringVar(
&opts.name, "name", "",
"New name for the machine",
)
cmd.Flags().StringVar(
&opts.publicIP, "public-ip", "",
fmt.Sprintf("Public IP address of the machine for ingress configuration. Use '%s' or '' to remove the public IP.", PublicIPNone),
)
cmd.Flags().StringVarP(
&opts.context, "context", "c", "",
"Name of the cluster context. (default is the current context)",
)
return cmd
}
func update(ctx context.Context, uncli *cli.CLI, cmd *cobra.Command, opts updateOptions, machineNameOrID string) error {
// Check if at least one flag was explicitly set
if !cmd.Flags().Changed("name") && !cmd.Flags().Changed("public-ip") {
return fmt.Errorf("at least one update flag must be specified (--name, --public-ip)")
}
client, err := uncli.ConnectCluster(ctx, opts.context)
if err != nil {
return err
}
defer client.Close()
// First, resolve the machine to get its ID
machine, err := client.InspectMachine(ctx, machineNameOrID)
if err != nil {
return fmt.Errorf("find machine: %w", err)
}
// Build the update request
req := &pb.UpdateMachineRequest{
MachineId: machine.Machine.Id,
}
if opts.name != "" {
req.Name = &opts.name
}
// Check if --public-ip flag was explicitly provided
if cmd.Flags().Changed("public-ip") {
if opts.publicIP == "" || opts.publicIP == PublicIPNone {
req.PublicIp = &pb.IP{} // Empty IP to signal removal
} else {
// Parse and validate the public IP
ip, err := netip.ParseAddr(opts.publicIP)
if err != nil {
return fmt.Errorf("invalid public IP address %q: %w", opts.publicIP, err)
}
req.PublicIp = pb.NewIP(ip)
}
}
// Perform the update operation
updatedMachine, err := client.UpdateMachine(ctx, req)
if err != nil {
return fmt.Errorf("update machine: %w", err)
}
// Report what was changed
changes := make([]string, 0)
if opts.name != "" {
changes = append(changes, fmt.Sprintf("name: %q -> %q", machine.Machine.Name, updatedMachine.Name))
}
if cmd.Flags().Changed("public-ip") {
oldIP := PublicIPNone
if machine.Machine.PublicIp != nil {
if addr, err := machine.Machine.PublicIp.ToAddr(); err == nil {
oldIP = addr.String()
}
}
newIP := PublicIPNone
if updatedMachine.PublicIp != nil {
if addr, err := updatedMachine.PublicIp.ToAddr(); err == nil {
newIP = addr.String()
}
}
changes = append(changes, fmt.Sprintf("public IP: %s -> %s", oldIP, newIP))
}
fmt.Printf("Machine %q (ID: %s) configuration updated:\n", updatedMachine.Name, updatedMachine.Id)
for _, change := range changes {
fmt.Printf(" %s\n", change)
}
return nil
}
+1
View File
@@ -9,6 +9,7 @@ import (
"github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/stringid"
"github.com/docker/go-units" "github.com/docker/go-units"
"github.com/psviderski/uncloud/internal/cli" "github.com/psviderski/uncloud/internal/cli"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
@@ -2,9 +2,7 @@
* *
# Allow files and directories. # Allow files and directories.
!blog/
!docs/ !docs/
!landing/
!src/ !src/
!static/ !static/
!pkg/ !pkg/
+1 -1
View File
@@ -5,5 +5,5 @@
:8000 { :8000 {
root * /usr/share/caddy root * /usr/share/caddy
file_server file_server
log try_files {path} /index.html
} }
+1 -4
View File
@@ -19,12 +19,9 @@ RUN npm ci
## Build the static site. ## Build the static site.
RUN npm run build RUN npm run build
# Stage 3: Serve static Docusaurus site and landing page with Caddy. # Stage 3: Serve static site with Caddy.
FROM caddy:2.10.0-alpine AS caddy FROM caddy:2.10.0-alpine AS caddy
## Copy the Caddyfile. ## Copy the Caddyfile.
COPY ./Caddyfile /etc/caddy/Caddyfile COPY ./Caddyfile /etc/caddy/Caddyfile
## Copy the Docusaurus build output. ## Copy the Docusaurus build output.
COPY --from=prod /opt/docusaurus/build /usr/share/caddy COPY --from=prod /opt/docusaurus/build /usr/share/caddy
# Copy the landing page assets.
COPY landing/images /usr/share/caddy/images
COPY landing/index.html landing/style.css /usr/share/caddy/
+41
View File
@@ -0,0 +1,41 @@
# Website
This website is built using [Docusaurus](https://docusaurus.io/), a modern static website generator.
### Installation
```
$ yarn
```
### Local Development
```
$ yarn start
```
This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server.
### Build
```
$ yarn build
```
This command generates static content into the `build` directory and can be served using any static contents hosting service.
### Deployment
Using SSH:
```
$ USE_SSH=true yarn deploy
```
Not using SSH:
```
$ GIT_USER=<Your GitHub username> yarn deploy
```
If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch.
@@ -1,31 +1,39 @@
--- # WireGuard overlay network for Docker containers
title: How to connect Docker containers across multiple hosts with WireGuard
description: Learn how to configure a WireGuard overlay network that lets Docker containers securely communicate
across multiple hosts. No exposed ports needed.
slug: connect-docker-containers-across-hosts-wireguard
image: ./wireguard-overlay.png
authors: psviderski
tags: [ docker, wireguard, networking, vpn ]
---
You want your Docker containers to talk to each other, but they're running on different machines. Perhaps across # How to connect Docker containers across multiple hosts using WireGuard
different cloud providers or mixing cloud with on-prem. The usual approach of mapping services to host ports quickly
becomes a pain. Worse, if they're on the public internet, you need to secure every exposed endpoint with TLS and auth.
What if your containers on different machines could communicate directly without exposing any ports? Using their private # Connect Docker containers across multiple hosts with WireGuard
Docker IPs, as if they were on the same machine. Here's how you can use pure WireGuard and some clever networking tricks
to make this work. You have Docker containers running on different Linux machines. You want container A on one machine to talk directly to
container B on another machine using their private IPs. For example, to run your application and database containers on
separate machines without exposing them publicly. Here's how you can use pure WireGuard and some networking tricks to
make this work.
I implemented this technique to enable cross-machine container communication in
[Uncloud](https://github.com/psviderski/uncloud), an open source clustering and deployment tool for Docker.
* [What we're building](#what-were-building)
* [Prequisites](#prerequisites)
* [Step 1: Configure Docker networks](#step-1-configure-docker-networks)
* [Step 2: Connect Docker networks with WireGuard](#step-2-connect-docker-networks-with-wireguard)
* [Step 3: Configure IP routing](#step-3-configure-ip-routing)
* [Step 4: Testing](#step-4-testing)
* [Step 5: Make the configuration persistent](#step-5-make-the-configuration-persistent)
* [Scaling beyond two machines and limitations](#scaling-beyond-two-machines-and-limitations)
* [Automating with Uncloud](#automating-with-uncloud)
* [Alternative solutions](#alternative-solutions)
* [Conclusion](#conclusion)
## What we're building ## What we're building
Docker containers are typically connected to a [bridge network](https://docs.docker.com/engine/network/drivers/bridge/) Docker containers are typically connected to a [bridge network](https://docs.docker.com/engine/network/drivers/bridge/)
on their host machine, which allows them to communicate with each other. A bridge network also provides isolation from on their host machine, which allows them to communicate with each other. A bridge network also provides isolation from
containers not connected to it and other networks on the host. What we want to achieve is connecting these bridge containers not connected to it and other networks on the host. What we want to achieve is to connect these bridge
networks across machines so that containers on different machines can communicate as if they were connected to the same networks across machines so that containers on different machines can communicate as if they were connected to the same
local bridge network. local bridge network.
The incantation we need is called a site-to-site VPN. Any solution would work. Moreover, if the machines are on the same The incantation we need is called a site-to-site VPN. Any solution would work. Moreover, if the machines are on the same
local network, they're already connected and only lack the appropriate routing configuration. But I'll describe a more local network, they're already connected and only miss the appropriate routing configuration. But I'll describe a more
versatile approach that works even when the machines are on different continents or behind NAT. WireGuard is the ideal versatile approach that works even when the machines are on different continents or behind NAT. WireGuard is the ideal
solution for this use case: it's lightweight, [fast](https://www.wireguard.com/performance/), simple to configure, solution for this use case: it's lightweight, [fast](https://www.wireguard.com/performance/), simple to configure,
provides [strong security](https://www.wireguard.com/protocol/) and NAT traversal. provides [strong security](https://www.wireguard.com/protocol/) and NAT traversal.
@@ -37,21 +45,19 @@ communicate with each other using their private IPs.
I will use these two machines: I will use these two machines:
* Machine 1: Debian 12 virtual machine in my homelab network in Australia, which is behind NAT * Machine 1: Debian 12 virtual machine in my homelab network in Australia which is behind NAT
* Machine 2: Ubuntu 24.04 server from Hetzner in Finland that has a public IP * Machine 2: Ubuntu 24.04 server from Hetzner in Finland that has a public IP
![WireGuard overlay network](wireguard-overlay.png) ![wireguard-overlay.png](wireguard-overlay.png)
<!-- truncate -->
## Prerequisites ## Prerequisites
- Basic knowledge of [Docker networking](https://docs.docker.com/network/) and [WireGuard](https://www.wireguard.com/). * Basic knowledge of [Docker networking](https://docs.docker.com/network/) and [WireGuard](https://www.wireguard.com/).
If you're new to these topics, you might want to read up on them first. If you're new to these topics, you might want to read up on them first.
- At least two Linux machines with root access and Docker installed. They should be on the same network or be able to * At least two Linux machines with root access and Docker installed. They should be on the same network or reachable
communicate over the internet. over the internet.
## Step 1: Configure Docker networks # Step 1: Configure Docker networks
Most of the commands in this guide require root privileges. You can run them with `sudo` or log in as root. I'll start Most of the commands in this guide require root privileges. You can run them with `sudo` or log in as root. I'll start
root shells on both machines with `sudo -i` for convenience. root shells on both machines with `sudo -i` for convenience.
@@ -64,7 +70,7 @@ Therefore, let's create new Docker bridge networks on each machine with manually
choose any subnets from choose any subnets from
the [private IPv4 address ranges](https://en.wikipedia.org/wiki/Private_network#Private_IPv4_addresses) the [private IPv4 address ranges](https://en.wikipedia.org/wiki/Private_network#Private_IPv4_addresses)
that do not overlap with each other or with your existing networks. I'll use `10.200.1.0/24` and `10.200.2.0/24` that do not overlap with each other or with your existing networks. I'll use `10.200.1.0/24` and `10.200.2.0/24`
for Machine 1 and Machine 2, respectively. They don't even need to be sequential or be part of the same larger network. for Machine 1 and Machine 2 respectively. They don't even need to be sequential or be part of the same larger network.
However, using a common parent network (like `10.200.0.0/16` in my case) can simplify firewall rules and make it easier However, using a common parent network (like `10.200.0.0/16` in my case) can simplify firewall rules and make it easier
to manage more machines later. to manage more machines later.
@@ -81,16 +87,16 @@ Starting with Docker 28.2.0 ([PR](https://github.com/moby/moby/pull/49832)), you
host interfaces you host interfaces you
allow [direct routing](https://docs.docker.com/engine/network/packet-filtering-firewalls/#direct-routing) to containers allow [direct routing](https://docs.docker.com/engine/network/packet-filtering-firewalls/#direct-routing) to containers
in bridge networks. This is done by specifying the `com.docker.network.bridge.trusted_host_interfaces` option when in bridge networks. This is done by specifying the `com.docker.network.bridge.trusted_host_interfaces` option when
creating the network. In our case, we want to allow routing via the WireGuard interface `wg0` that will be created in creating the network. In our case, we want to allow routing via the WireGuard interface `wg0` that we be created in the
the next step. next step.
Provide this option even if you're using an older Docker version, as it'll be required if you upgrade Docker in the Provide this option even if you're using an older Docker version as it'll be required if you upgrade Docker in the
future. future.
## Step 2: Connect Docker networks with WireGuard ## Step 2: Connect Docker networks with WireGuard
By default, WireGuard uses the UDP port 51820 for communication. To establish a tunnel, at least one of the machines By default, WireGuard uses the UDP port 51280 for communication. To establish a tunnel, at least one of the machines
needs to be able to reach the other's port over the internet or local network. Please make sure it's not blocked by a need to be able to reach the other's port over the internet or local network. Please make sure it's not blocked by a
firewall on both machines. firewall on both machines.
For example, when using `iptables`, you can allow incoming UDP traffic on port 51820 with the following command: For example, when using `iptables`, you can allow incoming UDP traffic on port 51820 with the following command:
@@ -122,7 +128,7 @@ PrivateKey = <replace with 'privatekey' file content from Machine 1>
[Peer] [Peer]
PublicKey = <replace with 'publickey' file content from Machine 2> PublicKey = <replace with 'publickey' file content from Machine 2>
# IP ranges for which a peer will route traffic: Docker subnet on Machine 2 # IP ranges for which a peer will route traffic - Docker subnet on Machine 2
AllowedIPs = 10.200.2.0/24 AllowedIPs = 10.200.2.0/24
# Public IP of Machine 2 # Public IP of Machine 2
Endpoint = 157.180.72.195:51820 Endpoint = 157.180.72.195:51820
@@ -139,7 +145,7 @@ PrivateKey = <replace with 'privatekey' file content from Machine 2>
[Peer] [Peer]
PublicKey = <replace with 'publickey' file content from Machine 1> PublicKey = <replace with 'publickey' file content from Machine 1>
# IP ranges for which a peer will route traffic: Docker subnet on Machine 1 # IP ranges for which a peer will route traffic - Docker subnet on Machine 1
AllowedIPs = 10.200.1.0/24 AllowedIPs = 10.200.1.0/24
# Reachable endpoint of Machine 1 # Reachable endpoint of Machine 1
# Endpoint = # Endpoint =
@@ -156,14 +162,14 @@ In my case, Machine 1 is behind NAT in my private homelab network which is not r
server (Machine 2). The bidirectional tunnel can still be established in this case but Machine 1 must initiate the server (Machine 2). The bidirectional tunnel can still be established in this case but Machine 1 must initiate the
connection. connection.
If both of your machines are reachable from each other, you should specify the `Endpoint` option in both configs which If both of your machine are reachable from each other, you should specify the `Endpoint` option in both configs which
will allow them to establish the connection without waiting for the other side to initiate it. If both of your machines will allow them to establish the connection without waiting for the other side to initiate it. If both of your machines
are behind NAT, see [NAT to NAT Connections](https://github.com/pirate/wireguard-docs#NAT-to-NAT-Connections) for more are behind NAT, see [NAT to NAT Connections](https://github.com/pirate/wireguard-docs#NAT-to-NAT-Connections) for more
information. information.
Note also that we don't set the `Address` option in the configs because we don't want to assign any IP addresses to the Note also that we don't set `Address` option in the configs because we don't want to assign any IP addresses to the
WireGuard interfaces. We want the tunnel to only encapsulate and transfer packets from the `multi-host` bridge networks WireGuard interfaces. We want the tunnel to only encapsulate and transfer packets from the `multi-host` bridge networks
and don't want either end of it to be the destination for the packets. and don't want any end of it to be the destination for the packets.
As the key pairs are now specified in the configuration files, you can remove the `privatekey` and `publickey` files on As the key pairs are now specified in the configuration files, you can remove the `privatekey` and `publickey` files on
both machines: both machines:
@@ -199,19 +205,14 @@ If you see the `latest handshake` time updating, it means the tunnel is working
## Step 3: Configure IP routing ## Step 3: Configure IP routing
You've established the WireGuard tunnel, but packets between containers won't flow yet. You need to configure IP routing
between the tunnel and the container networks.
Docker daemon automatically enables IP forwarding in the kernel when it starts, so you don't need to manually configure Docker daemon automatically enables IP forwarding in the kernel when it starts, so you don't need to manually configure
`net.ipv4.ip_forward` with `sysctl`. `net.ipv4.ip_forward` with `sysctl`.
The challenge is that Docker blocks traffic between external interfaces and container networks by default for security However, Docker blocks traffic between external interfaces and container networks by default for security. You need to
reasons. You need to explicitly allow WireGuard traffic from `wg0` interface to reach your containers via the explicitly allow WireGuard traffic from `wg0` interface to reach your containers via the `multi-host` bridge interface.
`multi-host` bridge interface. Docker uses iptables, so you can allow this traffic by adding a rule to the `FORWARD` Docker uses iptables, so you can allow this traffic by adding a rule to the `FORWARD` chain before any other
chain before any other Docker-managed rules that would drop it. Docker-managed rules that would drop it. Luckily, Docker creates a special `DOCKER-USER` chain exactly for this purpose
that the `FORWARD` chain jumps to before jumping to any other Docker-managed chains.
Fortunately, Docker creates a special `DOCKER-USER` chain exactly for this purpose. It's processed before other
Docker-managed chains, allowing you to add custom rules that won't be overridden by Docker.
To create the required iptables rule, you need to find the bridge interface name for the `multi-host` network you To create the required iptables rule, you need to find the bridge interface name for the `multi-host` network you
created earlier. It's named `br-<short-network-id>`, where `<short-network-id>` is the first 12 characters of the created earlier. It's named `br-<short-network-id>`, where `<short-network-id>` is the first 12 characters of the
@@ -219,7 +220,7 @@ network ID.
Add the iptables rule to allow traffic from `wg0` to `multi-host` bridge on Machine 1: Add the iptables rule to allow traffic from `wg0` to `multi-host` bridge on Machine 1:
```shell ```bash
$ docker network ls -f name=multi-host $ docker network ls -f name=multi-host
NETWORK ID NAME DRIVER SCOPE NETWORK ID NAME DRIVER SCOPE
661096b2a5d9 multi-host bridge local 661096b2a5d9 multi-host bridge local
@@ -228,14 +229,14 @@ $ iptables -I DOCKER-USER -i wg0 -o br-661096b2a5d9 -j ACCEPT
Add the iptables rule to allow traffic from `wg0` to `multi-host` bridge on Machine 2: Add the iptables rule to allow traffic from `wg0` to `multi-host` bridge on Machine 2:
```shell ```bash
$ docker network ls -f name=multi-host $ docker network ls -f name=multi-host
NETWORK ID NAME DRIVER SCOPE NETWORK ID NAME DRIVER SCOPE
48f808048e7c multi-host bridge local 48f808048e7c multi-host bridge local
$ iptables -I DOCKER-USER -i wg0 -o br-48f808048e7c -j ACCEPT $ iptables -I DOCKER-USER -i wg0 -o br-48f808048e7c -j ACCEPT
``` ```
The traffic in the other direction (from `multi-host` bridge to `wg0`) is not blocked by Docker by default. But it still The traffic the other way around (from `multi-host` bridge to `wg0`) is not blocked by Docker by default. But it still
won't be able to make it through the tunnel. The reason is that Docker creates a `MASQUERADE` rule in the `nat` table won't be able to make it through the tunnel. The reason is that Docker creates a `MASQUERADE` rule in the `nat` table
for every bridge network with option for every bridge network with option
[`com.docker.network.bridge.enable_ip_masquerade`](https://docs.docker.com/engine/network/drivers/bridge/#options) set [`com.docker.network.bridge.enable_ip_masquerade`](https://docs.docker.com/engine/network/drivers/bridge/#options) set
@@ -245,13 +246,13 @@ to `true` (which is the default). In my case, the rule looks like this on Machin
POSTROUTING -s 10.200.1.0/24 ! -o br-661096b2a5d9 -j MASQUERADE POSTROUTING -s 10.200.1.0/24 ! -o br-661096b2a5d9 -j MASQUERADE
``` ```
This essentially configures NAT for all external traffic coming from containers which is necessary for allowing them to This essentially configures NAT for all external traffic coming from containers which is necessary to allow them to
access the internet and other external networks. However, it equally applies to the traffic going through the `wg0` access the internet and other external networks. However, it equally applies to the traffic going through the `wg0`
interface. It tries to masquerade the source IP address of the packets with the IP address of the `wg0` interface and interface. It tries to masquerade the source IP address of the packets with the IP address of the `wg0` interface and
fails because the `wg0` interface doesn't have an IP. This results in the packets being fails because the `wg0` interface doesn't have an IP. This results in the packets being
[dropped](https://elixir.bootlin.com/linux/v6.15.5/source/net/netfilter/nf_nat_masquerade.c#L54-L58). [dropped](https://elixir.bootlin.com/linux/v6.15.5/source/net/netfilter/nf_nat_masquerade.c#L54-L58).
You could assign an IP address to `wg0` but this would cause the following unwanted side effects: You cloud assign an IP address to `wg0` but this would cause the following unwanted side effects:
- Containers from other Docker networks on the same machine could route through the tunnel to reach remote `multi-host` - Containers from other Docker networks on the same machine could route through the tunnel to reach remote `multi-host`
containers, violating Docker's network isolation model. containers, violating Docker's network isolation model.
@@ -321,7 +322,7 @@ PING 10.200.2.2 (10.200.2.2): 56 data bytes
64 bytes from 10.200.2.2: seq=2 ttl=62 time=297.285 ms 64 bytes from 10.200.2.2: seq=2 ttl=62 time=297.285 ms
``` ```
Both hosts have IPs assigned to the `multi-host` bridges, `10.200.1.1` and `10.200.2.1` respectively, which should also Both hosts have IPs assigned to the `multi-host` bridges, `10.200.1.1` and `10.200.2.1` respectively which should aslo
be reachable from the containers or hosts on both machines. be reachable from the containers or hosts on both machines.
You can see from the `ping` command the latency is quite high (~300 ms) in my case because the packets have to travel You can see from the `ping` command the latency is quite high (~300 ms) in my case because the packets have to travel
@@ -343,12 +344,12 @@ be to use `PostUp` and `PostDown` options in the WireGuard configs to automatica
starts/stops. starts/stops.
Append the following lines to the `[Interface]` section in `/etc/wireguard/wg0.conf`. Make sure to replace Append the following lines to the `[Interface]` section in `/etc/wireguard/wg0.conf`. Make sure to replace
`<network-id>` with your actual Docker network ID from [Step 3](#step-3-configure-ip-routing). The `%i` is replaced by `<network-id>` with your actual Docker network ID from Step 3. The `%i` is replaced by WireGuard with the interface
WireGuard with the interface name (`wg0`). name (`wg0`).
On Machine 1: On Machine 1:
```ini ```shell
[Interface] [Interface]
... ...
PostUp = iptables -I DOCKER-USER -i %i -o br-<network-id> -j ACCEPT; iptables -t nat -I POSTROUTING -s 10.200.1.0/24 -o %i -j RETURN PostUp = iptables -I DOCKER-USER -i %i -o br-<network-id> -j ACCEPT; iptables -t nat -I POSTROUTING -s 10.200.1.0/24 -o %i -j RETURN
@@ -357,7 +358,7 @@ PostDown = iptables -D DOCKER-USER -i %i -o br-<network-id> -j ACCEPT; iptables
On Machine 2: On Machine 2:
```ini ```shell
[Interface] [Interface]
... ...
PostUp = iptables -I DOCKER-USER -i %i -o br-<network-id> -j ACCEPT; iptables -t nat -I POSTROUTING -s 10.200.2.0/24 -o %i -j RETURN PostUp = iptables -I DOCKER-USER -i %i -o br-<network-id> -j ACCEPT; iptables -t nat -I POSTROUTING -s 10.200.2.0/24 -o %i -j RETURN
@@ -367,7 +368,7 @@ PostDown = iptables -D DOCKER-USER -i %i -o br-<network-id> -j ACCEPT; iptables
### Start WireGuard on boot ### Start WireGuard on boot
The `wireguard-tools` package provides a convenient systemd service to manage WireGuard interfaces. Since our iptables The `wireguard-tools` package provides a convenient systemd service to manage WireGuard interfaces. Since our iptables
rules should have priority over Docker's rules, WireGuard must start after Docker. rules should have a priority over Docker's rules, WireGuard must start after Docker.
Create a systemd drop-in configuration for this: Create a systemd drop-in configuration for this:
@@ -385,135 +386,54 @@ Then enable the WireGuard service to start on boot:
```shell ```shell
systemctl enable wg-quick@wg0.service systemctl enable wg-quick@wg0.service
systemctl daemon-reload systemctl daemon-reload
# Verify the unit includes the drop-in configuration # Verify the unit includes the drop-in configuration.
systemctl cat wg-quick@wg0.service systemctl cat wg-quick@wg0.service
``` ```
## Scaling beyond two machines ## Scaling beyond two machines and limitations
Adding a third machine means following the same steps as above on it and updating WireGuard configs on *all* existing //Adding a third machine requires updating configs on all existing machines. This gets tedious fast... //WireGuard mesh
machines. Each machine needs a `[Peer]` section for every other machine in the network. With 5 machines, that's 4 peer and challenges to manually manage key pairs and distribute configs //Requirements for NAT traversal: at least one
entries per config file or 20 peer configurations total that establish a full mesh topology. machine in each pair must be reachable by the other. The wireguard will fail to establish a connection if both machines
are behind NAT without special tricks that are beyond the scope of this post. DNS resolution for container names across
![WireGuard full mesh](wireguard-mesh.png) machines is not covered here, but you can use a service discovery tool like Consul.
## Limitations
### DNS resolution
The main limitation of this setup is that containers can't find each other by name across machines. You need to use
their IP addresses directly or implement a service discovery solution like Consul or CoreDNS.
For small deployments, you can assign static IPs to containers and use those IPs in your app configuration. But service
discovery is essential for larger and more dynamic deployments.
### NAT traversal constraints
For WireGuard connections to work, at least one machine in each pair must be publicly reachable. The connection fails if
both machines are behind NAT. While solutions exist (STUN/TURN servers, UDP hole punching), they're beyond the scope of
this guide.
Common scenarios that work:
- ✅ Cloud VPS (public or private IP) ↔ Cloud VPS (public or private IP). Both can use private IPs only if they're in the
same cloud provider's network
- ✅ Homelab (behind NAT) ↔ Cloud VPS (public IP)
- ✅ Homelab (private IP) ↔ Homelab (private IP on the same local network)
- ❌ Homelab (behind NAT) ↔ Friend's homelab (behind NAT) — requires a relay server
## Automating with Uncloud ## Automating with Uncloud
As your setup grows, managing subnet allocation for Docker networks (ensuring each gets a unique range like //I built Uncloud to handle all the heavy lifting automatically.
`10.200.1.0/24`, `10.200.2.0/24`) and updating WireGuard configs manually may become tedious quickly.
I built [Uncloud](https://github.com/psviderski/uncloud), an open source clustering and deployment tool for Docker, to You can initialise a cluster of machines by running the following commands:
handle all the heavy lifting automatically. You can get the same result and much more with just a few commands.
Initialise a new cluster on your first machine:
```shell ```shell
uc machine init user@machine1 uc machine init user@machine1-ip
uc machine add user@machine2-ip
...
uc machine add user@machineN-ip
``` ```
Add more machines to the cluster: //This will create `uncloud` Docker bridge network on each machine with `10.210.N.0/24` subnet by default and set up
//WireGuard mesh network between them and make persistent across reboots.
```shell //Mention embedded DNS that resolves container IPs by their service names and multi-machine Docker Compose support.
uc machine add user@machine2
uc machine add user@machine3
```
This is what these commands do:
- Create the `uncloud` Docker network on each machine with unique subnets (`10.210.0.0/24`, `10.210.1.0/24`, etc.).
- Generate WireGuard key pairs and distribute public keys across machines.
- Start a full mesh WireGuard network.
- Configure iptables rules for container communication.
- Make everything persistent across reboots.
Beyond the network setup, you also get:
- Multi-machine [Docker Compose](https://docs.docker.com/reference/compose-file/) deployments with zero downtime.
- Built-in DNS server that resolves container IPs by their service names.
- Automatic HTTPS and reverse proxy configuration.
Check out the [documentation](https://uncloud.run/docs) for more information.
## Alternative solutions ## Alternative solutions
Before settling on the WireGuard approach, I evaluated several alternatives. Note that I only considered lightweight //I wanted to explore only lightweight solutions for Docker so not talking about Kubernetes and a numerous CNI
solutions suitable for Docker. Kubernetes and its CNI ecosystem deserve a separate discussion. //drivers. Let's leave this beast for another time.
### Docker Swarm overlay network ### Docker Swarm overlay network
Docker Swarm includes built-in [overlay networking](https://docs.docker.com/engine/network/drivers/overlay/). However,
to use an overlay network, you need to run a [Swarm cluster](https://docs.docker.com/engine/swarm/) on all machines.
This introduces additional complexity:
- Cluster nodes must
[maintain the quorum](https://docs.docker.com/engine/swarm/admin_guide/#maintain-the-quorum-of-managers). Losing
quorum impacts the functionality of overlay networks.
- Ports 2377, 7946, and 4789 must be exposed to untrusted networks (if connecting machines over the internet) for
cluster management, node communication, and VXLAN overlay traffic.
- VXLAN traffic is unencrypted by default, requiring additional
[hardening](https://docs.docker.com/engine/swarm/swarm-tutorial/#open-protocols-and-ports-between-the-hosts) with
IPSec and firewalls.
- Every node must be publicly reachable. VXLAN fails if machines are behind NAT.
If these limitations are acceptable, an overlay network is a great option. Note that you can use an overlay network with
regular containers without using any other Swarm features.
### Flannel ### Flannel
[Flannel](https://github.com/flannel-io/flannel) is battle-tested in Kubernetes but can also be used with Docker. It
supports multiple [backends](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md) including
VXLAN and WireGuard.
The main caveat is that Flannel requires running etcd as the datastore for coordination. Depending on your availability
requirements, you may need to set up an etcd cluster with multiple nodes. This is not a problem if you're already using
Kubernetes. But if you're just running a few Docker hosts, it might seem like overkill.
### Tailscale ### Tailscale
[Tailscale](https://tailscale.com/) makes WireGuard easy with automatic NAT traversal and key management, but it's not //Not a generic site-to-site VPN, so the recommended approach is to use Tailscale on the container level. This way a
designed as a generic site-to-site VPN for connecting networks. Instead, it connects individual devices and provides //container that needs to talk across machines is configured as a Tailscale machine so it can connect to other Tailscale
identity-based access controls. //machines. Maybe the subnet router feature can be used to connect Docker networks in a similar I described here, but
//I haven't tested it.
The recommended approach for using [Tailscale with Docker](https://tailscale.com/kb/1282/docker) is to connect each
individual container to a Tailscale network. This means deploying an additional Tailscale container alongside every
application container.
Tailscale's [subnet router](https://tailscale.com/kb/1019/subnets) feature might work to expose Docker networks similar
to our setup, but I haven't tested this approach.
## Conclusion ## Conclusion
That's it! Now you know how to securely connect Docker containers across multiple machines using WireGuard. The manual //Summarise what we've done.?
setup works great for a handful of machines that you don't need to change often, but configuration management becomes
tedious as you scale.
If you don't want to mess with manual configuration, consider automation tools like Uncloud or evaluate if you need a If you like this article and my work, you can follow me on X [@psviderski](https://x.com/psviderski).
full orchestration platform.
Feel free to reach out if you have any questions or suggestions. You can find me on X
at [@psviderski](https://x.com/psviderski) or check my GitHub profile [psviderski](https://github.com/psviderski/)
for other contacts.

Before

Width:  |  Height:  |  Size: 897 KiB

After

Width:  |  Height:  |  Size: 897 KiB

+8
View File
@@ -0,0 +1,8 @@
services:
uncloud-docs:
image: ghcr.io/psviderski/uncloud-docs:latest
pull_policy: always
user: nobody
x-ports:
- docs.uncloud.run:8000/https
scale: 2
View File
@@ -89,23 +89,6 @@ For example, move it to `/usr/local/bin` which is a common location for user-ins
sudo mv ./uc /usr/local/bin sudo mv ./uc /usr/local/bin
``` ```
Follow the same steps to upgrade to the latest version in the future.
## Debian
On a Debian system, you can install Uncloud CLI from an unofficial
[repository](https://debian.griffo.io/) maintained by
[@dariogriffo](https://github.com/dariogriffo):
```shell
curl -sS https://debian.griffo.io/EA0F721D231FDD3A0A17B9AC7808B4DD62C41256.asc | sudo gpg --dearmor --yes -o /etc/apt/trusted.gpg.d/debian.griffo.io.gpg
echo "deb https://debian.griffo.io/apt $(lsb_release -sc 2>/dev/null) main" | sudo tee /etc/apt/sources.list.d/debian.griffo.io.list
apt install -y uncloud
```
Alternatively, you can download `.deb` packages directly from the repository
[releases](https://github.com/dariogriffo/uncloud-debian/releases) page.
## Verify installation ## Verify installation
After installation, verify that `uc` command is working: After installation, verify that `uc` command is working:
@@ -114,6 +97,22 @@ After installation, verify that `uc` command is working:
uc --version uc --version
``` ```
## Linux (via package managers)
### Debian
Via unofficial repository packages created and maintained at [uncloud-debian](https://github.com/dariogriffo/uncloud-debian/) by @dariogriffo
You can install uncloud the debian way by running:
```sh
curl -sS https://debian.griffo.io/EA0F721D231FDD3A0A17B9AC7808B4DD62C41256.asc | sudo gpg --dearmor --yes -o /etc/apt/trusted.gpg.d/debian.griffo.io.gpg
echo "deb https://debian.griffo.io/apt $(lsb_release -sc 2>/dev/null) main" | sudo tee /etc/apt/sources.list.d/debian.griffo.io.list
apt install -y uncloud
```
or in the releases page of the repository [here](https://github.com/dariogriffo/uncloud-debian/releases)
## Next steps ## Next steps
Now that you have `uc` installed, you're ready to: Now that you have `uc` installed, you're ready to:
@@ -9,8 +9,8 @@ infrastructure with secure internet access.
Before you begin, you'll need: Before you begin, you'll need:
- **Uncloud CLI** [installed](1-install-cli.md) on your local machine - **Uncloud CLI** [installed](1-install-cli.md) on your local machine
- A **Ubuntu or Debian server** with **public IP address** and **SSH access** using a **private key** (as `root` or a - A **Ubuntu or Debian server** with **public IP address** and **SSH access** (as `root` or a user with `sudo`
user with **passwordless** `sudo` privileges). privileges) using a **private key**.
:::tip Need a server? :::tip Need a server?
@@ -271,16 +271,6 @@ Want to use your own domain, for example, `excalidraw.example.com` instead of `e
Add a CNAME record `excalidraw.example.com` in your DNS provider (Cloudflare, Namecheap, etc.) pointing to Add a CNAME record `excalidraw.example.com` in your DNS provider (Cloudflare, Namecheap, etc.) pointing to
`excalidraw.7za6s7.cluster.uncloud.run`. Alternatively, you can add an A record pointing to your server's IP. `excalidraw.7za6s7.cluster.uncloud.run`. Alternatively, you can add an A record pointing to your server's IP.
:::info note
These instructions set up your own domain **in addition to** the Uncloud managed DNS name
`excalidraw.7za6s7.cluster.uncloud.run`.
If you want to avoid the managed service altogether, add `--no-dns` to your `uc machine init` command, and point an A
DNS record to your server(s)'s IP(s).
:::
Then update the published port `80/https` in `compose.yaml` to use your domain: Then update the published port `80/https` in `compose.yaml` to use your domain:
```yaml title="compose.yaml" ```yaml title="compose.yaml"

Before

Width:  |  Height:  |  Size: 389 KiB

After

Width:  |  Height:  |  Size: 389 KiB

@@ -10,12 +10,12 @@ import {themes as prismThemes} from 'prism-react-renderer';
/** @type {import('@docusaurus/types').Config} */ /** @type {import('@docusaurus/types').Config} */
const config = { const config = {
title: 'Uncloud', title: 'Uncloud Docs',
tagline: 'Self-host and scale web apps without Kubernetes complexity', tagline: 'Dinosaurs are cool',
favicon: 'img/favicon.png', favicon: 'img/favicon.png',
// Set the production url of your site here // Set the production url of your site here
url: 'https://uncloud.run', url: 'https://docs.uncloud.run',
// Set the /<baseUrl>/ pathname under which your site is served // Set the /<baseUrl>/ pathname under which your site is served
// For GitHub pages deployment, it is often '/<projectName>/' // For GitHub pages deployment, it is often '/<projectName>/'
baseUrl: '/', baseUrl: '/',
@@ -54,30 +54,27 @@ const config = {
({ ({
docs: { docs: {
// Remove this to remove the "edit this page" links. // Remove this to remove the "edit this page" links.
editUrl: 'https://github.com/psviderski/uncloud/edit/main/website/', editUrl: 'https://github.com/psviderski/uncloud/edit/main/docs/',
// Serve the docs at the site's root. // Serve the docs at the site's root.
// routeBasePath: '/', routeBasePath: '/',
showLastUpdateTime: true, showLastUpdateTime: true,
sidebarPath: './sidebars.js', sidebarPath: './sidebars.js',
}, },
blog: { // blog: {
blogDescription: 'Blog posts from the Uncloud team and community', // showReadingTime: true,
blogSidebarTitle: 'All posts', // feedOptions: {
blogSidebarCount: 'ALL', // type: ['rss', 'atom'],
showReadingTime: true, // xslt: true,
feedOptions: { // },
type: ['rss', 'atom'], // // Please change this to your repo.
xslt: true, // // Remove this to remove the "edit this page" links.
},
// Please change this to your repo.
// Remove this to remove the "edit this page" links.
// editUrl: // editUrl:
// 'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/', // 'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/',
// Useful options to enforce blogging best practices // // Useful options to enforce blogging best practices
onInlineTags: 'warn', // onInlineTags: 'warn',
onInlineAuthors: 'warn', // onInlineAuthors: 'warn',
onUntruncatedBlogPosts: 'warn', // onUntruncatedBlogPosts: 'warn',
}, // },
theme: { theme: {
customCss: './src/css/custom.css', customCss: './src/css/custom.css',
}, },
@@ -99,24 +96,12 @@ const config = {
// Relative to your site's "static" directory. Cannot be SVGs. Can be external URLs too. // Relative to your site's "static" directory. Cannot be SVGs. Can be external URLs too.
image: 'img/social-card.png', image: 'img/social-card.png',
navbar: { navbar: {
title: 'Uncloud', title: 'Uncloud Docs',
logo: { logo: {
alt: 'Uncloud Logo', alt: 'Uncloud Logo',
src: 'img/logo.svg', src: 'img/logo.svg',
href: 'https://uncloud.run',
}, },
items: [ items: [
{
type: 'doc',
docId: 'overview',
label: 'Docs',
position: 'left',
},
{
to: 'blog',
label: 'Blog',
position: 'left',
},
{ {
type: 'search', type: 'search',
position: 'right', position: 'right',
+18434
View File
File diff suppressed because it is too large Load Diff
+45
View File
@@ -0,0 +1,45 @@
{
"name": "docs",
"version": "0.0.0",
"private": true,
"scripts": {
"docusaurus": "docusaurus",
"start": "docusaurus start",
"build": "docusaurus build",
"swizzle": "docusaurus swizzle",
"deploy": "docusaurus deploy",
"clear": "docusaurus clear",
"serve": "docusaurus serve",
"write-translations": "docusaurus write-translations",
"write-heading-ids": "docusaurus write-heading-ids"
},
"dependencies": {
"@docusaurus/core": "3.7.0",
"@docusaurus/preset-classic": "3.7.0",
"@easyops-cn/docusaurus-search-local": "^0.49.2",
"@mdx-js/react": "^3.0.0",
"clsx": "^2.0.0",
"prism-react-renderer": "^2.3.0",
"react": "^19.0.0",
"react-dom": "^19.0.0"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "3.7.0",
"@docusaurus/types": "3.7.0"
},
"browserslist": {
"production": [
">0.5%",
"not dead",
"not op_mini all"
],
"development": [
"last 3 chrome version",
"last 3 firefox version",
"last 5 safari version"
]
},
"engines": {
"node": ">=18.0"
}
}

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

Before

Width:  |  Height:  |  Size: 341 B

After

Width:  |  Height:  |  Size: 341 B

Before

Width:  |  Height:  |  Size: 88 KiB

After

Width:  |  Height:  |  Size: 88 KiB

+1 -2
View File
@@ -20,7 +20,7 @@ Uncloud stores its configuration in `~/.config/uncloud/config.yaml`. If you wish
### Initialisation ### Initialisation
Begin by initialising the first node in your cluster with `uc machine init [USER@HOST:PORT]`. If you do not have a need for Caddy reverse proxy, you may disable this feature with `--no-caddy`. If you want to avoid using uncloud's managed DNS service, add the `--no-dns` flag. Begin by initialising the first node in your cluster with `uc machine init [USER@HOST:PORT]`. If you do not have a need for Caddy reverse proxy, you may disable this feature with `--no-caddy`.
This command will idempotently install Docker, uncloudd, uncloud-corrosion. If Caddy is enabled, it will set up a reverse proxy. If Uncloud DNS is enabled, it will create a DNS A record for the machine's public IP address under `*.[CLUSTER ID].cluster.uncloud.run`. This command will idempotently install Docker, uncloudd, uncloud-corrosion. If Caddy is enabled, it will set up a reverse proxy. If Uncloud DNS is enabled, it will create a DNS A record for the machine's public IP address under `*.[CLUSTER ID].cluster.uncloud.run`.
@@ -37,7 +37,6 @@ Uncloud (uncloud.run) DNS can be managed with the `uc dns` subcommand.
* To reserve a domain name, run `uc dns reserve` * To reserve a domain name, run `uc dns reserve`
* To release a domain name, run `uc dns release`. * To release a domain name, run `uc dns release`.
* To see the domain name, run `uc dns show` * To see the domain name, run `uc dns show`
* To avoid using the Uncloud managed DNS service, use the `--no-dns` flag on your `uc machine init` command.
### Running a service ### Running a service
+2 -3
View File
@@ -3,11 +3,10 @@ package main
import ( import (
"context" "context"
"fmt" "fmt"
"log/slog"
"time"
"github.com/hashicorp/serf/serf" "github.com/hashicorp/serf/serf"
crdt "github.com/ipfs/go-ds-crdt" crdt "github.com/ipfs/go-ds-crdt"
"log/slog"
"time"
) )
// Implements the Broadcaster interface. // Implements the Broadcaster interface.
+1 -2
View File
@@ -2,10 +2,9 @@ package main
import ( import (
"fmt" "fmt"
"github.com/dgraph-io/badger/v3"
"log" "log"
"time" "time"
"github.com/dgraph-io/badger/v3"
) )
func customTimeEncoder(t time.Time) string { func customTimeEncoder(t time.Time) string {
+1 -2
View File
@@ -2,10 +2,9 @@ package main
import ( import (
"fmt" "fmt"
"github.com/ipfs/go-log/v2"
"log/slog" "log/slog"
"os" "os"
"github.com/ipfs/go-log/v2"
) )
// ipfsLogger is an slog logger that implements the IPFS go-log StandardLogger interface. // ipfsLogger is an slog logger that implements the IPFS go-log StandardLogger interface.
+1 -2
View File
@@ -2,13 +2,12 @@ package main
import ( import (
"fmt" "fmt"
"github.com/docker/docker/libnetwork/networkdb"
"log/slog" "log/slog"
"os" "os"
"os/signal" "os/signal"
"syscall" "syscall"
"time" "time"
"github.com/docker/docker/libnetwork/networkdb"
) )
func main() { func main() {
+6 -7
View File
@@ -4,13 +4,6 @@ import (
"context" "context"
"flag" "flag"
"fmt" "fmt"
"log/slog"
"net"
"os"
"os/signal"
"syscall"
"time"
"github.com/hashicorp/memberlist" "github.com/hashicorp/memberlist"
"github.com/hashicorp/serf/cmd/serf/command/agent" "github.com/hashicorp/serf/cmd/serf/command/agent"
"github.com/hashicorp/serf/serf" "github.com/hashicorp/serf/serf"
@@ -18,6 +11,12 @@ import (
badger "github.com/ipfs/go-ds-badger3" badger "github.com/ipfs/go-ds-badger3"
crdt "github.com/ipfs/go-ds-crdt" crdt "github.com/ipfs/go-ds-crdt"
"github.com/lmittmann/tint" "github.com/lmittmann/tint"
"log/slog"
"net"
"os"
"os/signal"
"syscall"
"time"
) )
func createSerfAgentConfig(name, bindAddr, rpcAddr, profile string) *agent.Config { func createSerfAgentConfig(name, bindAddr, rpcAddr, profile string) *agent.Config {
+5 -4
View File
@@ -4,19 +4,18 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"log/slog"
"github.com/hashicorp/serf/serf" "github.com/hashicorp/serf/serf"
"github.com/ipfs/boxo/datastore/dshelp" "github.com/ipfs/boxo/datastore/dshelp"
dag "github.com/ipfs/boxo/ipld/merkledag" dag "github.com/ipfs/boxo/ipld/merkledag"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore" ds "github.com/ipfs/go-datastore"
ipld "github.com/ipfs/go-ipld-format" ipld "github.com/ipfs/go-ipld-format"
"log/slog"
) )
// Implements the DAGService interface. // Implements the DAGService interface.
// TODO: implement SessionDAGService to optimize node fetching. // TODO: implement SessionDAGService to optimize node fetching.
// TODO: persistentSerfDAG? // TOOD: persistentSerfDAG?
type dagSyncer struct { type dagSyncer struct {
// Persistent storage for the nodes. // Persistent storage for the nodes.
store ds.Datastore store ds.Datastore
@@ -51,7 +50,8 @@ func (d *dagSyncer) Get(ctx context.Context, cid cid.Cid) (ipld.Node, error) {
} }
slog.Debug("Queried node from peers", "cid", cid, "deadline", query.Deadline()) slog.Debug("Queried node from peers", "cid", cid, "deadline", query.Deadline())
for { responded := false
for !responded {
select { select {
case resp, ok := <-query.ResponseCh(): case resp, ok := <-query.ResponseCh():
if !ok { if !ok {
@@ -63,6 +63,7 @@ func (d *dagSyncer) Get(ctx context.Context, cid cid.Cid) (ipld.Node, error) {
continue continue
} }
slog.Debug("Received node from peer", "cid", cid, "peer", resp.From) slog.Debug("Received node from peer", "cid", cid, "peer", resp.From)
responded = true
query.Close() query.Close()
node, err = nodeFromBytes(resp.Payload) node, err = nodeFromBytes(resp.Payload)
+3 -4
View File
@@ -6,13 +6,12 @@ import (
"crypto/cipher" "crypto/cipher"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"net/netip"
"time"
"github.com/psviderski/uncloud/internal/machine/network"
"github.com/siderolabs/discovery-api/api/v1alpha1/client/pb" "github.com/siderolabs/discovery-api/api/v1alpha1/client/pb"
discovery "github.com/siderolabs/discovery-client/pkg/client" discovery "github.com/siderolabs/discovery-client/pkg/client"
"go.uber.org/zap" "go.uber.org/zap"
"net/netip"
"time"
"github.com/psviderski/uncloud/internal/machine/network"
) )
const ( const (
+1 -1
View File
@@ -35,7 +35,6 @@ require (
github.com/jmoiron/sqlx v1.4.0 github.com/jmoiron/sqlx v1.4.0
github.com/lmittmann/tint v1.0.5 github.com/lmittmann/tint v1.0.5
github.com/miekg/dns v1.1.65 github.com/miekg/dns v1.1.65
github.com/mitchellh/mapstructure v1.5.0
github.com/moby/term v0.5.0 github.com/moby/term v0.5.0
github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0 github.com/opencontainers/image-spec v1.1.0
@@ -205,6 +204,7 @@ require (
github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/buildkit v0.17.2 // indirect github.com/moby/buildkit v0.17.2 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect
+35 -35
View File
@@ -6,8 +6,8 @@ import (
"fmt" "fmt"
"net/netip" "net/netip"
"os" "os"
"slices"
"github.com/charmbracelet/huh"
"github.com/docker/cli/cli/streams" "github.com/docker/cli/cli/streams"
"github.com/psviderski/uncloud/internal/cli/config" "github.com/psviderski/uncloud/internal/cli/config"
"github.com/psviderski/uncloud/internal/fs" "github.com/psviderski/uncloud/internal/fs"
@@ -22,12 +22,7 @@ import (
"google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/emptypb"
) )
const ( const defaultContextName = "default"
// DefaultSSHKeyPath is the fallback location for the SSH private key when provisioning remote machines.
// Used when no key is explicitly provided and SSH agent authentication fails.
DefaultSSHKeyPath = "~/.ssh/id_ed25519"
defaultContextName = "default"
)
type CLI struct { type CLI struct {
Config *config.Config Config *config.Config
@@ -178,7 +173,7 @@ func (cli *CLI) initRemoteMachine(ctx context.Context, opts InitClusterOptions)
return nil, fmt.Errorf("cluster context '%s' already exists", contextName) return nil, fmt.Errorf("cluster context '%s' already exists", contextName)
} }
machineClient, err := provisionRemoteMachine(ctx, opts.RemoteMachine, opts.Version) machineClient, err := cli.provisionRemoteMachine(ctx, *opts.RemoteMachine, opts.Version)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -195,7 +190,7 @@ func (cli *CLI) initRemoteMachine(ctx context.Context, opts InitClusterOptions)
return nil, fmt.Errorf("inspect machine: %w", err) return nil, fmt.Errorf("inspect machine: %w", err)
} }
if minfo.Id != "" { if minfo.Id != "" {
if err = promptResetMachine(ctx, machineClient.MachineClient); err != nil { if err = cli.promptResetMachine(); err != nil {
return nil, err return nil, err
} }
} }
@@ -254,7 +249,7 @@ type AddMachineOptions struct {
Context string Context string
MachineName string MachineName string
PublicIP *netip.Addr PublicIP *netip.Addr
RemoteMachine *RemoteMachine RemoteMachine RemoteMachine
Version string Version string
} }
@@ -277,7 +272,7 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
} }
}() }()
machineClient, err := provisionRemoteMachine(ctx, opts.RemoteMachine, opts.Version) machineClient, err := cli.provisionRemoteMachine(ctx, opts.RemoteMachine, opts.Version)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@@ -293,18 +288,7 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
return nil, nil, fmt.Errorf("inspect machine: %w", err) return nil, nil, fmt.Errorf("inspect machine: %w", err)
} }
if minfo.Id != "" { if minfo.Id != "" {
// Check if the machine is already a member of this cluster. if err = cli.promptResetMachine(); err != nil {
machines, err := c.ListMachines(ctx, nil)
if err != nil {
return nil, nil, fmt.Errorf("list cluster machines: %w", err)
}
if slices.ContainsFunc(machines, func(m *pb.MachineMember) bool {
return m.Machine.Id == minfo.Id
}) {
return nil, nil, fmt.Errorf("machine is already a member of this cluster (%s)", minfo.Name)
}
if err = promptResetMachine(ctx, machineClient.MachineClient); err != nil {
return nil, nil, err return nil, nil, err
} }
} }
@@ -355,7 +339,7 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
return nil, nil, fmt.Errorf("add machine to cluster (context '%s'): %w", contextName, err) return nil, nil, fmt.Errorf("add machine to cluster (context '%s'): %w", contextName, err)
} }
// Get the most up-to-date list of other machines in the cluster to include them in the join request. // List other machines in the cluster to include them in the join request.
machines, err := c.ListMachines(ctx, nil) machines, err := c.ListMachines(ctx, nil)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("list cluster machines: %w", err) return nil, nil, fmt.Errorf("list cluster machines: %w", err)
@@ -398,20 +382,11 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
// provisionRemoteMachine installs the Uncloud daemon and dependencies on the remote machine over SSH and returns // provisionRemoteMachine installs the Uncloud daemon and dependencies on the remote machine over SSH and returns
// a machine API client to interact with the machine. The client should be closed after use by the caller. // a machine API client to interact with the machine. The client should be closed after use by the caller.
// The version parameter specifies the version of the Uncloud daemon to install. If empty, the latest version is used. // The version parameter specifies the version of the Uncloud daemon to install. If empty, the latest version is used.
// The remoteMachine.SSHKeyPath could be updated to the default SSH key path if it is not set and the SSH agent func (cli *CLI) provisionRemoteMachine(
// authentication fails. ctx context.Context, remoteMachine RemoteMachine, version string,
func provisionRemoteMachine(
ctx context.Context, remoteMachine *RemoteMachine, version string,
) (*client.Client, error) { ) (*client.Client, error) {
// Provision the remote machine by installing the Uncloud daemon and dependencies over SSH. // Provision the remote machine by installing the Uncloud daemon and dependencies over SSH.
sshClient, err := sshexec.Connect(remoteMachine.User, remoteMachine.Host, remoteMachine.Port, remoteMachine.KeyPath) sshClient, err := sshexec.Connect(remoteMachine.User, remoteMachine.Host, remoteMachine.Port, remoteMachine.KeyPath)
// If the SSH connection using SSH agent fails and no key path is provided, try to use the default SSH key.
if err != nil && remoteMachine.KeyPath == "" {
remoteMachine.KeyPath = DefaultSSHKeyPath
sshClient, err = sshexec.Connect(
remoteMachine.User, remoteMachine.Host, remoteMachine.Port, remoteMachine.KeyPath,
)
}
if err != nil { if err != nil {
return nil, fmt.Errorf( return nil, fmt.Errorf(
"SSH login to remote machine %s: %w", "SSH login to remote machine %s: %w",
@@ -445,6 +420,31 @@ func provisionRemoteMachine(
return machineClient, nil return machineClient, nil
} }
func (cli *CLI) promptResetMachine() error {
var confirm bool
form := huh.NewForm(
huh.NewGroup(
huh.NewConfirm().
Title(
"The remote machine is already initialised as a cluster member. Do you want to reset it first?",
).
Affirmative("Yes!").
Negative("No").
Value(&confirm),
),
).WithAccessible(true)
if err := form.Run(); err != nil {
return fmt.Errorf("prompt user to confirm: %w", err)
}
if !confirm {
return fmt.Errorf("remote machine is already initialised as a cluster member")
}
// TODO: implement resetting the remote machine.
return fmt.Errorf("resetting the remote machine is not implemented yet. " +
"Please manually run 'uncloud-uninstall' on the remote machine to fully uninstall Uncloud from it")
}
// ProgressOut returns an output stream for progress writer. // ProgressOut returns an output stream for progress writer.
func (cli *CLI) ProgressOut() *streams.Out { func (cli *CLI) ProgressOut() *streams.Out {
return streams.NewOut(os.Stdout) return streams.NewOut(os.Stdout)
+2 -2
View File
@@ -53,11 +53,11 @@ func (c *Config) Read() error {
func (c *Config) Save() error { func (c *Config) Save() error {
dir, _ := filepath.Split(c.path) dir, _ := filepath.Split(c.path)
if err := os.MkdirAll(dir, 0o700); err != nil { if err := os.MkdirAll(dir, 0700); err != nil {
return fmt.Errorf("create config directory '%s': %w", dir, err) return fmt.Errorf("create config directory '%s': %w", dir, err)
} }
f, err := os.OpenFile(c.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) f, err := os.OpenFile(c.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil { if err != nil {
return fmt.Errorf("write config file '%s': %w", c.path, err) return fmt.Errorf("write config file '%s': %w", c.path, err)
} }
+2 -83
View File
@@ -5,20 +5,12 @@ import (
"fmt" "fmt"
"os" "os"
"strings" "strings"
"time"
"github.com/cenkalti/backoff/v4"
"github.com/charmbracelet/huh"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/sshexec" "github.com/psviderski/uncloud/internal/sshexec"
"google.golang.org/protobuf/types/known/emptypb"
) )
const (
// TODO: support pinning the script version to the CLI version. // TODO: support pinning the script version to the CLI version.
installScriptURL = "https://raw.githubusercontent.com/psviderski/uncloud/refs/heads/main/scripts/install.sh" const installScriptURL = "https://raw.githubusercontent.com/psviderski/uncloud/refs/heads/main/scripts/install.sh"
rootUser = "root"
)
type RemoteMachine struct { type RemoteMachine struct {
User string User string
@@ -32,7 +24,7 @@ func installCmd(user string, version string) string {
var env []string var env []string
// Add the SSH user (non-root) to the uncloud group to allow access to the Uncloud daemon unix socket. // Add the SSH user (non-root) to the uncloud group to allow access to the Uncloud daemon unix socket.
if user != rootUser { if user != "root" {
sudoPrefix = "sudo" sudoPrefix = "sudo"
env = append(env, "UNCLOUD_GROUP_ADD_USER="+sshexec.Quote(user)) env = append(env, "UNCLOUD_GROUP_ADD_USER="+sshexec.Quote(user))
} }
@@ -54,26 +46,6 @@ func provisionMachine(ctx context.Context, exec sshexec.Executor, version string
return fmt.Errorf("run whoami: %w", err) return fmt.Errorf("run whoami: %w", err)
} }
if user != rootUser {
// 'sudo -n' is not used because it fails with 'sudo: a password is required' when the user has no password
// in /etc/shadow even though it may have valid sudo access.
out, err := exec.Run(ctx, "sudo true")
if err != nil {
if strings.Contains(out, "password is required") {
return fmt.Errorf(
"user '%[1]s' requires a password for sudo, but Uncloud needs passwordless sudo or root access "+
"to install and configure the uncloudd daemon on the remote machine.\n\n"+
"Possible solutions:\n"+
"1. Use root user or a user with passwordless sudo instead.\n"+
"2. Configure passwordless sudo for the user '%[1]s' by running on the remote machine:\n"+
" echo '%[1]s ALL=(ALL) NOPASSWD:ALL' | sudo tee /etc/sudoers.d/%[1]s",
user)
}
return fmt.Errorf("sudo command failed for user '%s': %w. "+
"Please ensure the user has sudo privileges or use root user instead", user, err)
}
}
cmd := installCmd(user, version) cmd := installCmd(user, version)
fmt.Println("Downloading Uncloud install script:", installScriptURL) fmt.Println("Downloading Uncloud install script:", installScriptURL)
@@ -84,56 +56,3 @@ func provisionMachine(ctx context.Context, exec sshexec.Executor, version string
} }
return nil return nil
} }
func promptResetMachine(ctx context.Context, machineClient pb.MachineClient) error {
var confirm bool
form := huh.NewForm(
huh.NewGroup(
huh.NewConfirm().
Title(
"The remote machine is already initialised as a cluster member. Do you want to reset it first?\n" +
"This will:\n" +
"- Remove all service containers from the machine\n" +
"- Reset the machine to the uninitialised state",
).
Affirmative("Yes!").
Negative("No").
Value(&confirm),
),
).WithAccessible(true)
if err := form.Run(); err != nil {
return fmt.Errorf("prompt user to confirm: %w", err)
}
if !confirm {
return fmt.Errorf("remote machine is already initialised as a cluster member")
}
if _, err := machineClient.Reset(ctx, &pb.ResetRequest{}); err != nil {
return fmt.Errorf("reset remote machine: %w. You can also manually run 'uncloud-uninstall' "+
"on the remote machine to fully uninstall Uncloud from it", err)
}
fmt.Println("Resetting the remote machine...")
if err := waitMachineReady(ctx, machineClient, 1*time.Minute); err != nil {
return fmt.Errorf("wait for machine to be ready after reset: %w", err)
}
return nil
}
// waitMachineReady waits for the machine to be ready to serve requests.
func waitMachineReady(ctx context.Context, machineClient pb.MachineClient, timeout time.Duration) error {
boff := backoff.WithContext(backoff.NewExponentialBackOff(
backoff.WithMaxInterval(1*time.Second),
backoff.WithMaxElapsedTime(timeout),
), ctx)
inspect := func() error {
_, err := machineClient.Inspect(ctx, &emptypb.Empty{})
if err != nil {
return fmt.Errorf("inspect machine: %w", err)
}
return nil
}
return backoff.Retry(inspect, boff)
}
+2 -3
View File
@@ -5,15 +5,14 @@ import (
"crypto/tls" "crypto/tls"
"errors" "errors"
"fmt" "fmt"
"github.com/cenkalti/backoff/v4"
"golang.org/x/net/http2"
"log/slog" "log/slog"
"net" "net"
"net/http" "net/http"
"net/netip" "net/netip"
"net/url" "net/url"
"time" "time"
"github.com/cenkalti/backoff/v4"
"golang.org/x/net/http2"
) )
const ( const (
+1 -2
View File
@@ -6,12 +6,11 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"github.com/cenkalti/backoff/v4"
"io" "io"
"log/slog" "log/slog"
"net/http" "net/http"
"strconv" "strconv"
"github.com/cenkalti/backoff/v4"
) )
type ChangeType string type ChangeType string
+1 -2
View File
@@ -3,9 +3,8 @@ package daemon
import ( import (
"context" "context"
"fmt" "fmt"
"log/slog"
systemd "github.com/coreos/go-systemd/daemon" systemd "github.com/coreos/go-systemd/daemon"
"log/slog"
"github.com/psviderski/uncloud/internal/machine" "github.com/psviderski/uncloud/internal/machine"
) )
+2 -3
View File
@@ -3,11 +3,10 @@ package daemon
import ( import (
"errors" "errors"
"fmt" "fmt"
"net/netip"
"os"
"github.com/psviderski/uncloud/internal/machine" "github.com/psviderski/uncloud/internal/machine"
"github.com/psviderski/uncloud/internal/machine/network" "github.com/psviderski/uncloud/internal/machine/network"
"net/netip"
"os"
) )
// MachineToken returns the local machine's token that can be used for adding the machine to a cluster. // MachineToken returns the local machine's token that can be used for adding the machine to a cluster.
+2 -3
View File
@@ -4,11 +4,10 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"log/slog"
"time"
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
"github.com/docker/docker/client" "github.com/docker/docker/client"
"log/slog"
"time"
) )
// WaitDaemonReady waits for the Docker daemon to start and be ready to serve requests. // WaitDaemonReady waits for the Docker daemon to start and be ready to serve requests.
+107 -350
View File
@@ -124,7 +124,7 @@ func (x DNSRecord_RecordType) Number() protoreflect.EnumNumber {
// Deprecated: Use DNSRecord_RecordType.Descriptor instead. // Deprecated: Use DNSRecord_RecordType.Descriptor instead.
func (DNSRecord_RecordType) EnumDescriptor() ([]byte, []int) { func (DNSRecord_RecordType) EnumDescriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{11, 0} return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{8, 0}
} }
type AddMachineRequest struct { type AddMachineRequest struct {
@@ -339,173 +339,6 @@ func (x *ListMachinesResponse) GetMachines() []*MachineMember {
return nil return nil
} }
type UpdateMachineRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Machine to update
MachineId string `protobuf:"bytes,1,opt,name=machine_id,json=machineId,proto3" json:"machine_id,omitempty"`
// Updated machine information
Name *string `protobuf:"bytes,2,opt,name=name,proto3,oneof" json:"name,omitempty"`
PublicIp *IP `protobuf:"bytes,3,opt,name=public_ip,json=publicIp,proto3,oneof" json:"public_ip,omitempty"`
Endpoints []*IPPort `protobuf:"bytes,4,rep,name=endpoints,proto3" json:"endpoints,omitempty"`
}
func (x *UpdateMachineRequest) Reset() {
*x = UpdateMachineRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *UpdateMachineRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*UpdateMachineRequest) ProtoMessage() {}
func (x *UpdateMachineRequest) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use UpdateMachineRequest.ProtoReflect.Descriptor instead.
func (*UpdateMachineRequest) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{4}
}
func (x *UpdateMachineRequest) GetMachineId() string {
if x != nil {
return x.MachineId
}
return ""
}
func (x *UpdateMachineRequest) GetName() string {
if x != nil && x.Name != nil {
return *x.Name
}
return ""
}
func (x *UpdateMachineRequest) GetPublicIp() *IP {
if x != nil {
return x.PublicIp
}
return nil
}
func (x *UpdateMachineRequest) GetEndpoints() []*IPPort {
if x != nil {
return x.Endpoints
}
return nil
}
type UpdateMachineResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Machine *MachineInfo `protobuf:"bytes,1,opt,name=machine,proto3" json:"machine,omitempty"`
}
func (x *UpdateMachineResponse) Reset() {
*x = UpdateMachineResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *UpdateMachineResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*UpdateMachineResponse) ProtoMessage() {}
func (x *UpdateMachineResponse) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[5]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use UpdateMachineResponse.ProtoReflect.Descriptor instead.
func (*UpdateMachineResponse) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{5}
}
func (x *UpdateMachineResponse) GetMachine() *MachineInfo {
if x != nil {
return x.Machine
}
return nil
}
type RemoveMachineRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
}
func (x *RemoveMachineRequest) Reset() {
*x = RemoveMachineRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *RemoveMachineRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*RemoveMachineRequest) ProtoMessage() {}
func (x *RemoveMachineRequest) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[6]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use RemoveMachineRequest.ProtoReflect.Descriptor instead.
func (*RemoveMachineRequest) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{6}
}
func (x *RemoveMachineRequest) GetId() string {
if x != nil {
return x.Id
}
return ""
}
type Domain struct { type Domain struct {
state protoimpl.MessageState state protoimpl.MessageState
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
@@ -517,7 +350,7 @@ type Domain struct {
func (x *Domain) Reset() { func (x *Domain) Reset() {
*x = Domain{} *x = Domain{}
if protoimpl.UnsafeEnabled { if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[7] mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
} }
@@ -530,7 +363,7 @@ func (x *Domain) String() string {
func (*Domain) ProtoMessage() {} func (*Domain) ProtoMessage() {}
func (x *Domain) ProtoReflect() protoreflect.Message { func (x *Domain) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[7] mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil { if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
@@ -543,7 +376,7 @@ func (x *Domain) ProtoReflect() protoreflect.Message {
// Deprecated: Use Domain.ProtoReflect.Descriptor instead. // Deprecated: Use Domain.ProtoReflect.Descriptor instead.
func (*Domain) Descriptor() ([]byte, []int) { func (*Domain) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{7} return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{4}
} }
func (x *Domain) GetName() string { func (x *Domain) GetName() string {
@@ -564,7 +397,7 @@ type ReserveDomainRequest struct {
func (x *ReserveDomainRequest) Reset() { func (x *ReserveDomainRequest) Reset() {
*x = ReserveDomainRequest{} *x = ReserveDomainRequest{}
if protoimpl.UnsafeEnabled { if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[8] mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
} }
@@ -577,7 +410,7 @@ func (x *ReserveDomainRequest) String() string {
func (*ReserveDomainRequest) ProtoMessage() {} func (*ReserveDomainRequest) ProtoMessage() {}
func (x *ReserveDomainRequest) ProtoReflect() protoreflect.Message { func (x *ReserveDomainRequest) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[8] mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[5]
if protoimpl.UnsafeEnabled && x != nil { if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
@@ -590,7 +423,7 @@ func (x *ReserveDomainRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use ReserveDomainRequest.ProtoReflect.Descriptor instead. // Deprecated: Use ReserveDomainRequest.ProtoReflect.Descriptor instead.
func (*ReserveDomainRequest) Descriptor() ([]byte, []int) { func (*ReserveDomainRequest) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{8} return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{5}
} }
func (x *ReserveDomainRequest) GetEndpoint() string { func (x *ReserveDomainRequest) GetEndpoint() string {
@@ -611,7 +444,7 @@ type CreateDomainRecordsRequest struct {
func (x *CreateDomainRecordsRequest) Reset() { func (x *CreateDomainRecordsRequest) Reset() {
*x = CreateDomainRecordsRequest{} *x = CreateDomainRecordsRequest{}
if protoimpl.UnsafeEnabled { if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[9] mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
} }
@@ -624,7 +457,7 @@ func (x *CreateDomainRecordsRequest) String() string {
func (*CreateDomainRecordsRequest) ProtoMessage() {} func (*CreateDomainRecordsRequest) ProtoMessage() {}
func (x *CreateDomainRecordsRequest) ProtoReflect() protoreflect.Message { func (x *CreateDomainRecordsRequest) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[9] mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[6]
if protoimpl.UnsafeEnabled && x != nil { if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
@@ -637,7 +470,7 @@ func (x *CreateDomainRecordsRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use CreateDomainRecordsRequest.ProtoReflect.Descriptor instead. // Deprecated: Use CreateDomainRecordsRequest.ProtoReflect.Descriptor instead.
func (*CreateDomainRecordsRequest) Descriptor() ([]byte, []int) { func (*CreateDomainRecordsRequest) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{9} return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{6}
} }
func (x *CreateDomainRecordsRequest) GetRecords() []*DNSRecord { func (x *CreateDomainRecordsRequest) GetRecords() []*DNSRecord {
@@ -658,7 +491,7 @@ type CreateDomainRecordsResponse struct {
func (x *CreateDomainRecordsResponse) Reset() { func (x *CreateDomainRecordsResponse) Reset() {
*x = CreateDomainRecordsResponse{} *x = CreateDomainRecordsResponse{}
if protoimpl.UnsafeEnabled { if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[10] mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
} }
@@ -671,7 +504,7 @@ func (x *CreateDomainRecordsResponse) String() string {
func (*CreateDomainRecordsResponse) ProtoMessage() {} func (*CreateDomainRecordsResponse) ProtoMessage() {}
func (x *CreateDomainRecordsResponse) ProtoReflect() protoreflect.Message { func (x *CreateDomainRecordsResponse) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[10] mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[7]
if protoimpl.UnsafeEnabled && x != nil { if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
@@ -684,7 +517,7 @@ func (x *CreateDomainRecordsResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use CreateDomainRecordsResponse.ProtoReflect.Descriptor instead. // Deprecated: Use CreateDomainRecordsResponse.ProtoReflect.Descriptor instead.
func (*CreateDomainRecordsResponse) Descriptor() ([]byte, []int) { func (*CreateDomainRecordsResponse) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{10} return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{7}
} }
func (x *CreateDomainRecordsResponse) GetRecords() []*DNSRecord { func (x *CreateDomainRecordsResponse) GetRecords() []*DNSRecord {
@@ -707,7 +540,7 @@ type DNSRecord struct {
func (x *DNSRecord) Reset() { func (x *DNSRecord) Reset() {
*x = DNSRecord{} *x = DNSRecord{}
if protoimpl.UnsafeEnabled { if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[11] mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
} }
@@ -720,7 +553,7 @@ func (x *DNSRecord) String() string {
func (*DNSRecord) ProtoMessage() {} func (*DNSRecord) ProtoMessage() {}
func (x *DNSRecord) ProtoReflect() protoreflect.Message { func (x *DNSRecord) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[11] mi := &file_internal_machine_api_pb_cluster_proto_msgTypes[8]
if protoimpl.UnsafeEnabled && x != nil { if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
@@ -733,7 +566,7 @@ func (x *DNSRecord) ProtoReflect() protoreflect.Message {
// Deprecated: Use DNSRecord.ProtoReflect.Descriptor instead. // Deprecated: Use DNSRecord.ProtoReflect.Descriptor instead.
func (*DNSRecord) Descriptor() ([]byte, []int) { func (*DNSRecord) Descriptor() ([]byte, []int) {
return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{11} return file_internal_machine_api_pb_cluster_proto_rawDescGZIP(), []int{8}
} }
func (x *DNSRecord) GetName() string { func (x *DNSRecord) GetName() string {
@@ -797,87 +630,59 @@ var file_internal_machine_api_pb_cluster_proto_rawDesc = []byte{
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73,
0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63,
0x68, 0x69, 0x6e, 0x65, 0x4d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x52, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x68, 0x69, 0x6e, 0x65, 0x4d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x52, 0x08, 0x6d, 0x61, 0x63, 0x68,
0x69, 0x6e, 0x65, 0x73, 0x22, 0xbb, 0x01, 0x0a, 0x14, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x69, 0x6e, 0x65, 0x73, 0x22, 0x1c, 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x12,
0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61,
0x0a, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x6d, 0x65, 0x22, 0x32, 0x0a, 0x14, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x44, 0x6f, 0x6d,
0x09, 0x52, 0x09, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x04, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x04, 0x6e, 0x61, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e,
0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x29, 0x0a, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x22, 0x46, 0x0a, 0x1a, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65,
0x69, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x07, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71,
0x50, 0x48, 0x01, 0x52, 0x08, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x70, 0x88, 0x01, 0x01, 0x75, 0x65, 0x73, 0x74, 0x12, 0x28, 0x0a, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18,
0x12, 0x29, 0x0a, 0x09, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x4e, 0x53, 0x52,
0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x50, 0x50, 0x6f, 0x72, 0x74, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x47,
0x52, 0x09, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x0a, 0x1b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65,
0x6e, 0x61, 0x6d, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a,
0x69, 0x70, 0x22, 0x43, 0x0a, 0x15, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e,
0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07,
0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x96, 0x01, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x52,
0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0x26, 0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70,
0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x4e,
0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x53, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79,
0x1c, 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75,
0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x32, 0x0a, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73,
0x14, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x22, 0x2e, 0x0a, 0x0a, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f,
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12,
0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x05, 0x0a, 0x01, 0x41, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x41, 0x41, 0x41, 0x10, 0x02,
0x74, 0x22, 0x46, 0x0a, 0x1a, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x32, 0x86, 0x03, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x3d, 0x0a, 0x0a,
0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x64, 0x64, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x16, 0x2e, 0x61, 0x70, 0x69,
0x28, 0x0a, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x2e, 0x41, 0x64, 0x64, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
0x32, 0x0e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x41, 0x64, 0x64, 0x4d, 0x61, 0x63, 0x68,
0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x47, 0x0a, 0x1b, 0x43, 0x72, 0x65, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0c, 0x4c,
0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x07, 0x72, 0x65, 0x63, 0x6f,
0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x61, 0x70, 0x69, 0x2e,
0x44, 0x4e, 0x53, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72,
0x64, 0x73, 0x22, 0x96, 0x01, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01,
0x28, 0x0e, 0x32, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x63, 0x6f,
0x72, 0x64, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74,
0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x03, 0x20,
0x03, 0x28, 0x09, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x22, 0x2e, 0x0a, 0x0a, 0x52,
0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53,
0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x05, 0x0a, 0x01, 0x41, 0x10,
0x01, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x41, 0x41, 0x41, 0x10, 0x02, 0x32, 0x92, 0x04, 0x0a, 0x07,
0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x3d, 0x0a, 0x0a, 0x41, 0x64, 0x64, 0x4d, 0x61,
0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x41, 0x64, 0x64, 0x4d,
0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e,
0x61, 0x70, 0x69, 0x2e, 0x41, 0x64, 0x64, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61,
0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x19,
0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x0d, 0x55, 0x70, 0x64,
0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x19, 0x2e, 0x61, 0x70, 0x69,
0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x55, 0x70, 0x64, 0x61,
0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x12, 0x42, 0x0a, 0x0d, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69,
0x6e, 0x65, 0x12, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4d,
0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e,
0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x37, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65,
0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x65, 0x73,
0x65, 0x72, 0x76, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x30,
0x0a, 0x09, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d,
0x70, 0x74, 0x79, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x70, 0x74, 0x79, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61,
0x12, 0x34, 0x0a, 0x0d, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37,
0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12,
0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x44, 0x6f, 0x6d,
0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x58, 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69,
0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x12, 0x1f, 0x2e, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x30, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x44, 0x6f,
0x61, 0x70, 0x69, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x0b, 0x2e, 0x61,
0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x70, 0x69, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x34, 0x0a, 0x0d, 0x52, 0x65, 0x6c,
0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x65, 0x61, 0x73, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f,
0x42, 0x37, 0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70,
0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x6b, 0x69, 0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x74, 0x79, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12,
0x64, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x58, 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52,
0x6e, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x12, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x72, 0x65,
0x33, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x72,
0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x37, 0x5a, 0x35, 0x67, 0x69, 0x74,
0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73,
0x6b, 0x69, 0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72,
0x6e, 0x61, 0x6c, 0x2f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f,
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
} }
var ( var (
@@ -893,7 +698,7 @@ func file_internal_machine_api_pb_cluster_proto_rawDescGZIP() []byte {
} }
var file_internal_machine_api_pb_cluster_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_internal_machine_api_pb_cluster_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
var file_internal_machine_api_pb_cluster_proto_msgTypes = make([]protoimpl.MessageInfo, 12) var file_internal_machine_api_pb_cluster_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
var file_internal_machine_api_pb_cluster_proto_goTypes = []any{ var file_internal_machine_api_pb_cluster_proto_goTypes = []any{
(MachineMember_MembershipState)(0), // 0: api.MachineMember.MembershipState (MachineMember_MembershipState)(0), // 0: api.MachineMember.MembershipState
(DNSRecord_RecordType)(0), // 1: api.DNSRecord.RecordType (DNSRecord_RecordType)(0), // 1: api.DNSRecord.RecordType
@@ -901,54 +706,43 @@ var file_internal_machine_api_pb_cluster_proto_goTypes = []any{
(*AddMachineResponse)(nil), // 3: api.AddMachineResponse (*AddMachineResponse)(nil), // 3: api.AddMachineResponse
(*MachineMember)(nil), // 4: api.MachineMember (*MachineMember)(nil), // 4: api.MachineMember
(*ListMachinesResponse)(nil), // 5: api.ListMachinesResponse (*ListMachinesResponse)(nil), // 5: api.ListMachinesResponse
(*UpdateMachineRequest)(nil), // 6: api.UpdateMachineRequest (*Domain)(nil), // 6: api.Domain
(*UpdateMachineResponse)(nil), // 7: api.UpdateMachineResponse (*ReserveDomainRequest)(nil), // 7: api.ReserveDomainRequest
(*RemoveMachineRequest)(nil), // 8: api.RemoveMachineRequest (*CreateDomainRecordsRequest)(nil), // 8: api.CreateDomainRecordsRequest
(*Domain)(nil), // 9: api.Domain (*CreateDomainRecordsResponse)(nil), // 9: api.CreateDomainRecordsResponse
(*ReserveDomainRequest)(nil), // 10: api.ReserveDomainRequest (*DNSRecord)(nil), // 10: api.DNSRecord
(*CreateDomainRecordsRequest)(nil), // 11: api.CreateDomainRecordsRequest (*NetworkConfig)(nil), // 11: api.NetworkConfig
(*CreateDomainRecordsResponse)(nil), // 12: api.CreateDomainRecordsResponse (*IP)(nil), // 12: api.IP
(*DNSRecord)(nil), // 13: api.DNSRecord (*MachineInfo)(nil), // 13: api.MachineInfo
(*NetworkConfig)(nil), // 14: api.NetworkConfig (*emptypb.Empty)(nil), // 14: google.protobuf.Empty
(*IP)(nil), // 15: api.IP
(*MachineInfo)(nil), // 16: api.MachineInfo
(*IPPort)(nil), // 17: api.IPPort
(*emptypb.Empty)(nil), // 18: google.protobuf.Empty
} }
var file_internal_machine_api_pb_cluster_proto_depIdxs = []int32{ var file_internal_machine_api_pb_cluster_proto_depIdxs = []int32{
14, // 0: api.AddMachineRequest.network:type_name -> api.NetworkConfig 11, // 0: api.AddMachineRequest.network:type_name -> api.NetworkConfig
15, // 1: api.AddMachineRequest.public_ip:type_name -> api.IP 12, // 1: api.AddMachineRequest.public_ip:type_name -> api.IP
16, // 2: api.AddMachineResponse.machine:type_name -> api.MachineInfo 13, // 2: api.AddMachineResponse.machine:type_name -> api.MachineInfo
16, // 3: api.MachineMember.machine:type_name -> api.MachineInfo 13, // 3: api.MachineMember.machine:type_name -> api.MachineInfo
0, // 4: api.MachineMember.state:type_name -> api.MachineMember.MembershipState 0, // 4: api.MachineMember.state:type_name -> api.MachineMember.MembershipState
4, // 5: api.ListMachinesResponse.machines:type_name -> api.MachineMember 4, // 5: api.ListMachinesResponse.machines:type_name -> api.MachineMember
15, // 6: api.UpdateMachineRequest.public_ip:type_name -> api.IP 10, // 6: api.CreateDomainRecordsRequest.records:type_name -> api.DNSRecord
17, // 7: api.UpdateMachineRequest.endpoints:type_name -> api.IPPort 10, // 7: api.CreateDomainRecordsResponse.records:type_name -> api.DNSRecord
16, // 8: api.UpdateMachineResponse.machine:type_name -> api.MachineInfo 1, // 8: api.DNSRecord.type:type_name -> api.DNSRecord.RecordType
13, // 9: api.CreateDomainRecordsRequest.records:type_name -> api.DNSRecord 2, // 9: api.Cluster.AddMachine:input_type -> api.AddMachineRequest
13, // 10: api.CreateDomainRecordsResponse.records:type_name -> api.DNSRecord 14, // 10: api.Cluster.ListMachines:input_type -> google.protobuf.Empty
1, // 11: api.DNSRecord.type:type_name -> api.DNSRecord.RecordType 7, // 11: api.Cluster.ReserveDomain:input_type -> api.ReserveDomainRequest
2, // 12: api.Cluster.AddMachine:input_type -> api.AddMachineRequest 14, // 12: api.Cluster.GetDomain:input_type -> google.protobuf.Empty
18, // 13: api.Cluster.ListMachines:input_type -> google.protobuf.Empty 14, // 13: api.Cluster.ReleaseDomain:input_type -> google.protobuf.Empty
6, // 14: api.Cluster.UpdateMachine:input_type -> api.UpdateMachineRequest 8, // 14: api.Cluster.CreateDomainRecords:input_type -> api.CreateDomainRecordsRequest
8, // 15: api.Cluster.RemoveMachine:input_type -> api.RemoveMachineRequest 3, // 15: api.Cluster.AddMachine:output_type -> api.AddMachineResponse
10, // 16: api.Cluster.ReserveDomain:input_type -> api.ReserveDomainRequest 5, // 16: api.Cluster.ListMachines:output_type -> api.ListMachinesResponse
18, // 17: api.Cluster.GetDomain:input_type -> google.protobuf.Empty 6, // 17: api.Cluster.ReserveDomain:output_type -> api.Domain
18, // 18: api.Cluster.ReleaseDomain:input_type -> google.protobuf.Empty 6, // 18: api.Cluster.GetDomain:output_type -> api.Domain
11, // 19: api.Cluster.CreateDomainRecords:input_type -> api.CreateDomainRecordsRequest 6, // 19: api.Cluster.ReleaseDomain:output_type -> api.Domain
3, // 20: api.Cluster.AddMachine:output_type -> api.AddMachineResponse 9, // 20: api.Cluster.CreateDomainRecords:output_type -> api.CreateDomainRecordsResponse
5, // 21: api.Cluster.ListMachines:output_type -> api.ListMachinesResponse 15, // [15:21] is the sub-list for method output_type
7, // 22: api.Cluster.UpdateMachine:output_type -> api.UpdateMachineResponse 9, // [9:15] is the sub-list for method input_type
18, // 23: api.Cluster.RemoveMachine:output_type -> google.protobuf.Empty 9, // [9:9] is the sub-list for extension type_name
9, // 24: api.Cluster.ReserveDomain:output_type -> api.Domain 9, // [9:9] is the sub-list for extension extendee
9, // 25: api.Cluster.GetDomain:output_type -> api.Domain 0, // [0:9] is the sub-list for field type_name
9, // 26: api.Cluster.ReleaseDomain:output_type -> api.Domain
12, // 27: api.Cluster.CreateDomainRecords:output_type -> api.CreateDomainRecordsResponse
20, // [20:28] is the sub-list for method output_type
12, // [12:20] is the sub-list for method input_type
12, // [12:12] is the sub-list for extension type_name
12, // [12:12] is the sub-list for extension extendee
0, // [0:12] is the sub-list for field type_name
} }
func init() { file_internal_machine_api_pb_cluster_proto_init() } func init() { file_internal_machine_api_pb_cluster_proto_init() }
@@ -1008,42 +802,6 @@ func file_internal_machine_api_pb_cluster_proto_init() {
} }
} }
file_internal_machine_api_pb_cluster_proto_msgTypes[4].Exporter = func(v any, i int) any { file_internal_machine_api_pb_cluster_proto_msgTypes[4].Exporter = func(v any, i int) any {
switch v := v.(*UpdateMachineRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[5].Exporter = func(v any, i int) any {
switch v := v.(*UpdateMachineResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[6].Exporter = func(v any, i int) any {
switch v := v.(*RemoveMachineRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_internal_machine_api_pb_cluster_proto_msgTypes[7].Exporter = func(v any, i int) any {
switch v := v.(*Domain); i { switch v := v.(*Domain); i {
case 0: case 0:
return &v.state return &v.state
@@ -1055,7 +813,7 @@ func file_internal_machine_api_pb_cluster_proto_init() {
return nil return nil
} }
} }
file_internal_machine_api_pb_cluster_proto_msgTypes[8].Exporter = func(v any, i int) any { file_internal_machine_api_pb_cluster_proto_msgTypes[5].Exporter = func(v any, i int) any {
switch v := v.(*ReserveDomainRequest); i { switch v := v.(*ReserveDomainRequest); i {
case 0: case 0:
return &v.state return &v.state
@@ -1067,7 +825,7 @@ func file_internal_machine_api_pb_cluster_proto_init() {
return nil return nil
} }
} }
file_internal_machine_api_pb_cluster_proto_msgTypes[9].Exporter = func(v any, i int) any { file_internal_machine_api_pb_cluster_proto_msgTypes[6].Exporter = func(v any, i int) any {
switch v := v.(*CreateDomainRecordsRequest); i { switch v := v.(*CreateDomainRecordsRequest); i {
case 0: case 0:
return &v.state return &v.state
@@ -1079,7 +837,7 @@ func file_internal_machine_api_pb_cluster_proto_init() {
return nil return nil
} }
} }
file_internal_machine_api_pb_cluster_proto_msgTypes[10].Exporter = func(v any, i int) any { file_internal_machine_api_pb_cluster_proto_msgTypes[7].Exporter = func(v any, i int) any {
switch v := v.(*CreateDomainRecordsResponse); i { switch v := v.(*CreateDomainRecordsResponse); i {
case 0: case 0:
return &v.state return &v.state
@@ -1091,7 +849,7 @@ func file_internal_machine_api_pb_cluster_proto_init() {
return nil return nil
} }
} }
file_internal_machine_api_pb_cluster_proto_msgTypes[11].Exporter = func(v any, i int) any { file_internal_machine_api_pb_cluster_proto_msgTypes[8].Exporter = func(v any, i int) any {
switch v := v.(*DNSRecord); i { switch v := v.(*DNSRecord); i {
case 0: case 0:
return &v.state return &v.state
@@ -1104,14 +862,13 @@ func file_internal_machine_api_pb_cluster_proto_init() {
} }
} }
} }
file_internal_machine_api_pb_cluster_proto_msgTypes[4].OneofWrappers = []any{}
type x struct{} type x struct{}
out := protoimpl.TypeBuilder{ out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{ File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(), GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_internal_machine_api_pb_cluster_proto_rawDesc, RawDescriptor: file_internal_machine_api_pb_cluster_proto_rawDesc,
NumEnums: 2, NumEnums: 2,
NumMessages: 12, NumMessages: 9,
NumExtensions: 0, NumExtensions: 0,
NumServices: 1, NumServices: 1,
}, },
-20
View File
@@ -11,8 +11,6 @@ import "internal/machine/api/pb/machine.proto";
service Cluster { service Cluster {
rpc AddMachine(AddMachineRequest) returns (AddMachineResponse); rpc AddMachine(AddMachineRequest) returns (AddMachineResponse);
rpc ListMachines(google.protobuf.Empty) returns (ListMachinesResponse); rpc ListMachines(google.protobuf.Empty) returns (ListMachinesResponse);
rpc UpdateMachine(UpdateMachineRequest) returns (UpdateMachineResponse);
rpc RemoveMachine(RemoveMachineRequest) returns (google.protobuf.Empty);
rpc ReserveDomain(ReserveDomainRequest) returns (Domain); rpc ReserveDomain(ReserveDomainRequest) returns (Domain);
rpc GetDomain(google.protobuf.Empty) returns (Domain); rpc GetDomain(google.protobuf.Empty) returns (Domain);
@@ -51,24 +49,6 @@ message ListMachinesResponse {
repeated MachineMember machines = 1; repeated MachineMember machines = 1;
} }
message UpdateMachineRequest {
// Machine to update
string machine_id = 1;
// Updated machine information
optional string name = 2;
optional IP public_ip = 3;
repeated IPPort endpoints = 4;
}
message UpdateMachineResponse {
MachineInfo machine = 1;
}
message RemoveMachineRequest {
string id = 1;
}
message Domain { message Domain {
string name = 1; string name = 1;
} }
@@ -22,8 +22,6 @@ const _ = grpc.SupportPackageIsVersion9
const ( const (
Cluster_AddMachine_FullMethodName = "/api.Cluster/AddMachine" Cluster_AddMachine_FullMethodName = "/api.Cluster/AddMachine"
Cluster_ListMachines_FullMethodName = "/api.Cluster/ListMachines" Cluster_ListMachines_FullMethodName = "/api.Cluster/ListMachines"
Cluster_UpdateMachine_FullMethodName = "/api.Cluster/UpdateMachine"
Cluster_RemoveMachine_FullMethodName = "/api.Cluster/RemoveMachine"
Cluster_ReserveDomain_FullMethodName = "/api.Cluster/ReserveDomain" Cluster_ReserveDomain_FullMethodName = "/api.Cluster/ReserveDomain"
Cluster_GetDomain_FullMethodName = "/api.Cluster/GetDomain" Cluster_GetDomain_FullMethodName = "/api.Cluster/GetDomain"
Cluster_ReleaseDomain_FullMethodName = "/api.Cluster/ReleaseDomain" Cluster_ReleaseDomain_FullMethodName = "/api.Cluster/ReleaseDomain"
@@ -36,8 +34,6 @@ const (
type ClusterClient interface { type ClusterClient interface {
AddMachine(ctx context.Context, in *AddMachineRequest, opts ...grpc.CallOption) (*AddMachineResponse, error) AddMachine(ctx context.Context, in *AddMachineRequest, opts ...grpc.CallOption) (*AddMachineResponse, error)
ListMachines(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ListMachinesResponse, error) ListMachines(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ListMachinesResponse, error)
UpdateMachine(ctx context.Context, in *UpdateMachineRequest, opts ...grpc.CallOption) (*UpdateMachineResponse, error)
RemoveMachine(ctx context.Context, in *RemoveMachineRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
ReserveDomain(ctx context.Context, in *ReserveDomainRequest, opts ...grpc.CallOption) (*Domain, error) ReserveDomain(ctx context.Context, in *ReserveDomainRequest, opts ...grpc.CallOption) (*Domain, error)
GetDomain(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*Domain, error) GetDomain(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*Domain, error)
ReleaseDomain(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*Domain, error) ReleaseDomain(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*Domain, error)
@@ -72,26 +68,6 @@ func (c *clusterClient) ListMachines(ctx context.Context, in *emptypb.Empty, opt
return out, nil return out, nil
} }
func (c *clusterClient) UpdateMachine(ctx context.Context, in *UpdateMachineRequest, opts ...grpc.CallOption) (*UpdateMachineResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(UpdateMachineResponse)
err := c.cc.Invoke(ctx, Cluster_UpdateMachine_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *clusterClient) RemoveMachine(ctx context.Context, in *RemoveMachineRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(emptypb.Empty)
err := c.cc.Invoke(ctx, Cluster_RemoveMachine_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *clusterClient) ReserveDomain(ctx context.Context, in *ReserveDomainRequest, opts ...grpc.CallOption) (*Domain, error) { func (c *clusterClient) ReserveDomain(ctx context.Context, in *ReserveDomainRequest, opts ...grpc.CallOption) (*Domain, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(Domain) out := new(Domain)
@@ -138,8 +114,6 @@ func (c *clusterClient) CreateDomainRecords(ctx context.Context, in *CreateDomai
type ClusterServer interface { type ClusterServer interface {
AddMachine(context.Context, *AddMachineRequest) (*AddMachineResponse, error) AddMachine(context.Context, *AddMachineRequest) (*AddMachineResponse, error)
ListMachines(context.Context, *emptypb.Empty) (*ListMachinesResponse, error) ListMachines(context.Context, *emptypb.Empty) (*ListMachinesResponse, error)
UpdateMachine(context.Context, *UpdateMachineRequest) (*UpdateMachineResponse, error)
RemoveMachine(context.Context, *RemoveMachineRequest) (*emptypb.Empty, error)
ReserveDomain(context.Context, *ReserveDomainRequest) (*Domain, error) ReserveDomain(context.Context, *ReserveDomainRequest) (*Domain, error)
GetDomain(context.Context, *emptypb.Empty) (*Domain, error) GetDomain(context.Context, *emptypb.Empty) (*Domain, error)
ReleaseDomain(context.Context, *emptypb.Empty) (*Domain, error) ReleaseDomain(context.Context, *emptypb.Empty) (*Domain, error)
@@ -160,12 +134,6 @@ func (UnimplementedClusterServer) AddMachine(context.Context, *AddMachineRequest
func (UnimplementedClusterServer) ListMachines(context.Context, *emptypb.Empty) (*ListMachinesResponse, error) { func (UnimplementedClusterServer) ListMachines(context.Context, *emptypb.Empty) (*ListMachinesResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListMachines not implemented") return nil, status.Errorf(codes.Unimplemented, "method ListMachines not implemented")
} }
func (UnimplementedClusterServer) UpdateMachine(context.Context, *UpdateMachineRequest) (*UpdateMachineResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method UpdateMachine not implemented")
}
func (UnimplementedClusterServer) RemoveMachine(context.Context, *RemoveMachineRequest) (*emptypb.Empty, error) {
return nil, status.Errorf(codes.Unimplemented, "method RemoveMachine not implemented")
}
func (UnimplementedClusterServer) ReserveDomain(context.Context, *ReserveDomainRequest) (*Domain, error) { func (UnimplementedClusterServer) ReserveDomain(context.Context, *ReserveDomainRequest) (*Domain, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReserveDomain not implemented") return nil, status.Errorf(codes.Unimplemented, "method ReserveDomain not implemented")
} }
@@ -235,42 +203,6 @@ func _Cluster_ListMachines_Handler(srv interface{}, ctx context.Context, dec fun
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
func _Cluster_UpdateMachine_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(UpdateMachineRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ClusterServer).UpdateMachine(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: Cluster_UpdateMachine_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ClusterServer).UpdateMachine(ctx, req.(*UpdateMachineRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Cluster_RemoveMachine_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RemoveMachineRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ClusterServer).RemoveMachine(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: Cluster_RemoveMachine_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ClusterServer).RemoveMachine(ctx, req.(*RemoveMachineRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Cluster_ReserveDomain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { func _Cluster_ReserveDomain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReserveDomainRequest) in := new(ReserveDomainRequest)
if err := dec(in); err != nil { if err := dec(in); err != nil {
@@ -358,14 +290,6 @@ var Cluster_ServiceDesc = grpc.ServiceDesc{
MethodName: "ListMachines", MethodName: "ListMachines",
Handler: _Cluster_ListMachines_Handler, Handler: _Cluster_ListMachines_Handler,
}, },
{
MethodName: "UpdateMachine",
Handler: _Cluster_UpdateMachine_Handler,
},
{
MethodName: "RemoveMachine",
Handler: _Cluster_RemoveMachine_Handler,
},
{ {
MethodName: "ReserveDomain", MethodName: "ReserveDomain",
Handler: _Cluster_ReserveDomain_Handler, Handler: _Cluster_ReserveDomain_Handler,
+1 -1
View File
@@ -14,7 +14,7 @@ service Machine {
rpc JoinCluster(JoinClusterRequest) returns (google.protobuf.Empty); rpc JoinCluster(JoinClusterRequest) returns (google.protobuf.Empty);
rpc Token(google.protobuf.Empty) returns (TokenResponse); rpc Token(google.protobuf.Empty) returns (TokenResponse);
rpc Inspect(google.protobuf.Empty) returns (MachineInfo); rpc Inspect(google.protobuf.Empty) returns (MachineInfo);
// Reset restores the machine to a clean state, removing all cluster-related configuration and data. // Reset restores the machine to a clean state, removing all cluster-related сonfiguration and data.
rpc Reset(ResetRequest) returns (google.protobuf.Empty); rpc Reset(ResetRequest) returns (google.protobuf.Empty);
rpc InspectService(InspectServiceRequest) returns (InspectServiceResponse); rpc InspectService(InspectServiceRequest) returns (InspectServiceResponse);
+2 -2
View File
@@ -39,7 +39,7 @@ type MachineClient interface {
JoinCluster(ctx context.Context, in *JoinClusterRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) JoinCluster(ctx context.Context, in *JoinClusterRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
Token(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*TokenResponse, error) Token(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*TokenResponse, error)
Inspect(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*MachineInfo, error) Inspect(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*MachineInfo, error)
// Reset restores the machine to a clean state, removing all cluster-related configuration and data. // Reset restores the machine to a clean state, removing all cluster-related сonfiguration and data.
Reset(ctx context.Context, in *ResetRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) Reset(ctx context.Context, in *ResetRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
InspectService(ctx context.Context, in *InspectServiceRequest, opts ...grpc.CallOption) (*InspectServiceResponse, error) InspectService(ctx context.Context, in *InspectServiceRequest, opts ...grpc.CallOption) (*InspectServiceResponse, error)
} }
@@ -132,7 +132,7 @@ type MachineServer interface {
JoinCluster(context.Context, *JoinClusterRequest) (*emptypb.Empty, error) JoinCluster(context.Context, *JoinClusterRequest) (*emptypb.Empty, error)
Token(context.Context, *emptypb.Empty) (*TokenResponse, error) Token(context.Context, *emptypb.Empty) (*TokenResponse, error)
Inspect(context.Context, *emptypb.Empty) (*MachineInfo, error) Inspect(context.Context, *emptypb.Empty) (*MachineInfo, error)
// Reset restores the machine to a clean state, removing all cluster-related configuration and data. // Reset restores the machine to a clean state, removing all cluster-related сonfiguration and data.
Reset(context.Context, *ResetRequest) (*emptypb.Empty, error) Reset(context.Context, *ResetRequest) (*emptypb.Empty, error)
InspectService(context.Context, *InspectServiceRequest) (*InspectServiceResponse, error) InspectService(context.Context, *InspectServiceRequest) (*InspectServiceResponse, error)
mustEmbedUnimplementedMachineServer() mustEmbedUnimplementedMachineServer()
+1 -2
View File
@@ -2,11 +2,10 @@ package proxy
import ( import (
"fmt" "fmt"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
"google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/encoding/protowire"
"google.golang.org/protobuf/proto" "google.golang.org/protobuf/proto"
"github.com/psviderski/uncloud/internal/machine/api/pb"
) )
// One2ManyResponder converts upstream responses into messages from upstreams, so that multiple // One2ManyResponder converts upstream responses into messages from upstreams, so that multiple
+1 -2
View File
@@ -2,12 +2,11 @@ package proxy
import ( import (
"context" "context"
"sync"
"github.com/siderolabs/grpc-proxy/proxy" "github.com/siderolabs/grpc-proxy/proxy"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata" "google.golang.org/grpc/metadata"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
"sync"
) )
// Director manages routing of gRPC requests between local and remote backends. // Director manages routing of gRPC requests between local and remote backends.
+1 -2
View File
@@ -2,12 +2,11 @@ package proxy
import ( import (
"context" "context"
"sync"
"github.com/siderolabs/grpc-proxy/proxy" "github.com/siderolabs/grpc-proxy/proxy"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/metadata" "google.golang.org/grpc/metadata"
"sync"
) )
// LocalBackend is a proxy.One2ManyResponder implementation that proxies to a local gRPC server listening on a Unix socket. // LocalBackend is a proxy.One2ManyResponder implementation that proxies to a local gRPC server listening on a Unix socket.
+3 -4
View File
@@ -3,15 +3,14 @@ package proxy
import ( import (
"context" "context"
"fmt" "fmt"
"net/netip"
"sync"
"time"
"github.com/siderolabs/grpc-proxy/proxy" "github.com/siderolabs/grpc-proxy/proxy"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/backoff" "google.golang.org/grpc/backoff"
"google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/metadata" "google.golang.org/grpc/metadata"
"net/netip"
"sync"
"time"
) )
// RemoteBackend is a proxy.One2ManyResponder implementation that proxies to a remote gRPC server, injecting machine metadata // RemoteBackend is a proxy.One2ManyResponder implementation that proxies to a remote gRPC server, injecting machine metadata
-17
View File
@@ -1,17 +0,0 @@
package caddyconfig
import (
"fmt"
"github.com/psviderski/uncloud/pkg/api"
)
func GenerateCaddyfile(containers []api.ServiceContainer, verifyResponse string) (string, error) {
return fmt.Sprintf(`http:// {
handle %s {
respond "%s" 200
}
log
}
`, VerifyPath, verifyResponse), nil
}
@@ -19,7 +19,7 @@ import (
"github.com/psviderski/uncloud/pkg/api" "github.com/psviderski/uncloud/pkg/api"
) )
func GenerateJSONConfig(containers []api.ServiceContainer, verifyResponse string) (*caddy.Config, error) { func GenerateConfig(containers []api.ServiceContainer, verifyResponse string) (*caddy.Config, error) {
// Maps hostnames to lists of upstreams (container IP:port pairs). // Maps hostnames to lists of upstreams (container IP:port pairs).
httpHostUpstreams := make(map[string][]string) httpHostUpstreams := make(map[string][]string)
httpsHostUpstreams := make(map[string][]string) httpsHostUpstreams := make(map[string][]string)
@@ -378,7 +378,7 @@ func TestGenerateConfig(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
config, err := GenerateJSONConfig(tt.containers, "verification-response-body") config, err := GenerateConfig(tt.containers, "verification-response-body")
if tt.wantErr { if tt.wantErr {
assert.Error(t, err) assert.Error(t, err)
+27 -51
View File
@@ -23,24 +23,23 @@ const (
// network. // network.
type Controller struct { type Controller struct {
store *store.Store store *store.Store
configDir string path string
verifyResponse string verifyResponse string
log *slog.Logger
} }
func NewController(store *store.Store, configDir string, verifyResponse string) (*Controller, error) { func NewController(store *store.Store, path string, verifyResponse string) (*Controller, error) {
if err := os.MkdirAll(configDir, 0o750); err != nil { dir := filepath.Dir(path)
return nil, fmt.Errorf("create directory for Caddy configuration '%s': %w", configDir, err) if err := os.MkdirAll(dir, 0750); err != nil {
return nil, fmt.Errorf("create parent directory for Caddy configuration '%s': %w", dir, err)
} }
if err := fs.Chown(configDir, "", CaddyGroup); err != nil { if err := fs.Chown(dir, "", CaddyGroup); err != nil {
return nil, fmt.Errorf("change owner of directory for Caddy configuration '%s': %w", configDir, err) return nil, fmt.Errorf("change owner of parent directory for Caddy configuration '%s': %w", dir, err)
} }
return &Controller{ return &Controller{
store: store, store: store,
configDir: configDir, path: path,
verifyResponse: verifyResponse, verifyResponse: verifyResponse,
log: slog.With("component", "caddy-controller"),
}, nil }, nil
} }
@@ -49,18 +48,14 @@ func (c *Controller) Run(ctx context.Context) error {
if err != nil { if err != nil {
return fmt.Errorf("subscribe to container changes: %w", err) return fmt.Errorf("subscribe to container changes: %w", err)
} }
c.log.Info("Subscribed to container changes in the cluster to generate Caddy configuration.") slog.Info("Subscribed to container changes in the cluster to generate Caddy configuration.")
containers, err := c.filterAvailableContainers(containerRecords) containers, err := c.filterAvailableContainers(containerRecords)
if err != nil { if err != nil {
return fmt.Errorf("filter available containers: %w", err) return fmt.Errorf("filter available containers: %w", err)
} }
if err = c.generateConfig(containers); err != nil {
if err = c.generateCaddyfile(containers); err != nil { return fmt.Errorf("generate Caddy configuration: %w", err)
return fmt.Errorf("generate Caddyfile configuration: %w", err)
}
if err = c.generateJSONConfig(containers); err != nil {
return fmt.Errorf("generate Caddy JSON configuration: %w", err)
} }
for { for {
@@ -69,27 +64,23 @@ func (c *Controller) Run(ctx context.Context) error {
if !ok { if !ok {
return fmt.Errorf("containers subscription failed") return fmt.Errorf("containers subscription failed")
} }
c.log.Info("Cluster containers changed, updating Caddy configuration.") slog.Debug("Cluster containers changed, updating Caddy configuration.")
containerRecords, err = c.store.ListContainers(ctx, store.ListOptions{}) containerRecords, err = c.store.ListContainers(ctx, store.ListOptions{})
if err != nil { if err != nil {
c.log.Info("Failed to list containers.", "err", err) slog.Error("Failed to list containers.", "err", err)
continue continue
} }
containers, err = c.filterAvailableContainers(containerRecords) containers, err = c.filterAvailableContainers(containerRecords)
if err != nil { if err != nil {
c.log.Info("Failed to filter available containers.", "err", err) slog.Error("Failed to filter available containers.", "err", err)
continue continue
} }
if err = c.generateConfig(containers); err != nil {
if err = c.generateCaddyfile(containers); err != nil { slog.Error("Failed to generate Caddy configuration.", "err", err)
c.log.Info("Failed to generate Caddyfile configuration.", "err", err)
}
if err = c.generateJSONConfig(containers); err != nil {
c.log.Info("Failed to generate Caddy JSON configuration.", "err", err)
} }
c.log.Info("Updated Caddy configuration.", "dir", c.configDir) slog.Debug("Updated Caddy configuration.", "path", c.path)
case <-ctx.Done(): case <-ctx.Done():
return nil return nil
} }
@@ -104,30 +95,16 @@ func (c *Controller) filterAvailableContainers(
) ([]api.ServiceContainer, error) { ) ([]api.ServiceContainer, error) {
containers := make([]api.ServiceContainer, len(containerRecords)) containers := make([]api.ServiceContainer, len(containerRecords))
for i, cr := range containerRecords { for i, cr := range containerRecords {
containers[i] = cr.Container containers[i] = api.ServiceContainer{
Container: cr.Container,
// TODO: restore ServiceSpec from the container record once it's saved in the store.
}
} }
return containers, nil return containers, nil
} }
func (c *Controller) generateCaddyfile(containers []api.ServiceContainer) error { func (c *Controller) generateConfig(containers []api.ServiceContainer) error {
caddyfile, err := GenerateCaddyfile(containers, c.verifyResponse) config, err := GenerateConfig(containers, c.verifyResponse)
if err != nil {
return fmt.Errorf("generate Caddyfile: %w", err)
}
caddyfilePath := filepath.Join(c.configDir, "Caddyfile")
if err = os.WriteFile(caddyfilePath, []byte(caddyfile), 0o640); err != nil {
return fmt.Errorf("write Caddyfile to file '%s': %w", caddyfilePath, err)
}
if err = fs.Chown(caddyfilePath, "", CaddyGroup); err != nil {
return fmt.Errorf("change owner of Caddyfile '%s': %w", caddyfilePath, err)
}
return nil
}
func (c *Controller) generateJSONConfig(containers []api.ServiceContainer) error {
config, err := GenerateJSONConfig(containers, c.verifyResponse)
if err != nil { if err != nil {
return err return err
} }
@@ -136,13 +113,12 @@ func (c *Controller) generateJSONConfig(containers []api.ServiceContainer) error
if err != nil { if err != nil {
return fmt.Errorf("marshal Caddy configuration: %w", err) return fmt.Errorf("marshal Caddy configuration: %w", err)
} }
configPath := filepath.Join(c.configDir, "caddy.json")
if err = os.WriteFile(configPath, configBytes, 0o640); err != nil { if err = os.WriteFile(c.path, configBytes, 0640); err != nil {
return fmt.Errorf("write Caddy configuration to file '%s': %w", configPath, err) return fmt.Errorf("write Caddy configuration to file '%s': %w", c.path, err)
} }
if err = fs.Chown(configPath, "", CaddyGroup); err != nil { if err = fs.Chown(c.path, "", CaddyGroup); err != nil {
return fmt.Errorf("change owner of Caddy configuration file '%s': %w", configPath, err) return fmt.Errorf("change owner of Caddy configuration file '%s': %w", c.path, err)
} }
return nil return nil
+3 -108
View File
@@ -5,18 +5,17 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/emptypb"
"log/slog" "log/slog"
"net/netip" "net/netip"
"time" "time"
"github.com/psviderski/uncloud/internal/corrosion" "github.com/psviderski/uncloud/internal/corrosion"
"github.com/psviderski/uncloud/internal/machine/api/pb" "github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/network" "github.com/psviderski/uncloud/internal/machine/network"
"github.com/psviderski/uncloud/internal/machine/store" "github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/internal/secret" "github.com/psviderski/uncloud/internal/secret"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/emptypb"
) )
type Cluster struct { type Cluster struct {
@@ -198,89 +197,6 @@ func (c *Cluster) AddMachine(ctx context.Context, req *pb.AddMachineRequest) (*p
return resp, nil return resp, nil
} }
// UpdateMachine updates machine configuration in the cluster.
func (c *Cluster) UpdateMachine(ctx context.Context, req *pb.UpdateMachineRequest) (*pb.UpdateMachineResponse, error) {
if err := c.checkInitialised(ctx); err != nil {
return nil, err
}
if req.MachineId == "" {
return nil, status.Error(codes.InvalidArgument, "machine_id not set")
}
// Get the current machine info
currentMachine, err := c.store.GetMachine(ctx, req.MachineId)
if err != nil {
if errors.Is(err, store.ErrMachineNotFound) {
return nil, status.Errorf(codes.NotFound, "machine not found: %s", req.MachineId)
}
return nil, status.Errorf(codes.Internal, "failed to get machine: %v", err)
}
// Create a copy of the current machine for updating
updatedMachine := &pb.MachineInfo{
Id: currentMachine.Id,
Name: currentMachine.Name,
Network: currentMachine.Network,
PublicIp: currentMachine.PublicIp,
}
// Apply updates from the request
if req.Name != nil {
// Check for empty name
if *req.Name == "" {
return nil, status.Error(codes.InvalidArgument, "machine name cannot be empty")
}
// Check for duplicate names (excluding the current machine)
if *req.Name != currentMachine.Name {
machines, err := c.store.ListMachines(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "list machines: %v", err)
}
for _, m := range machines {
if m.Id != req.MachineId && m.Name == *req.Name {
return nil, status.Errorf(codes.AlreadyExists, "machine with name %q already exists", *req.Name)
}
}
}
updatedMachine.Name = *req.Name
}
if req.PublicIp != nil {
// Check if this is an empty IP (used to signal removal)
if len(req.PublicIp.Ip) == 0 {
// User wants to remove public IP
updatedMachine.PublicIp = nil
} else {
// Validate and set the new IP
ip, err := req.PublicIp.ToAddr()
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid public IP: %v", err)
}
if !ip.IsValid() {
return nil, status.Error(codes.InvalidArgument, "invalid public IP")
}
updatedMachine.PublicIp = req.PublicIp
}
}
if req.Endpoints != nil {
updatedMachine.Network.Endpoints = req.Endpoints
}
// Update the machine in the store
if err = c.store.UpdateMachine(ctx, updatedMachine); err != nil {
if errors.Is(err, store.ErrMachineNotFound) {
return nil, status.Errorf(codes.NotFound, "machine not found: %s", req.MachineId)
}
return nil, status.Errorf(codes.Internal, "update machine: %v", err)
}
slog.Info("Machine configuration updated in the cluster.",
"id", updatedMachine.Id, "name", updatedMachine.Name)
resp := &pb.UpdateMachineResponse{Machine: updatedMachine}
return resp, nil
}
// ListMachines lists all machines in the cluster including their membership states. // ListMachines lists all machines in the cluster including their membership states.
func (c *Cluster) ListMachines(ctx context.Context, _ *emptypb.Empty) (*pb.ListMachinesResponse, error) { func (c *Cluster) ListMachines(ctx context.Context, _ *emptypb.Empty) (*pb.ListMachinesResponse, error) {
if err := c.checkInitialised(ctx); err != nil { if err := c.checkInitialised(ctx); err != nil {
@@ -326,24 +242,3 @@ func (c *Cluster) ListMachines(ctx context.Context, _ *emptypb.Empty) (*pb.ListM
return &pb.ListMachinesResponse{Machines: members}, nil return &pb.ListMachinesResponse{Machines: members}, nil
} }
// RemoveMachine removes a machine from the cluster.
func (c *Cluster) RemoveMachine(ctx context.Context, req *pb.RemoveMachineRequest) (*emptypb.Empty, error) {
if err := c.checkInitialised(ctx); err != nil {
return nil, err
}
if req.Id == "" {
return nil, status.Error(codes.InvalidArgument, "machine ID not set")
}
if err := c.store.DeleteMachine(ctx, req.Id); err != nil {
if errors.Is(err, store.ErrMachineNotFound) {
return nil, status.Errorf(codes.NotFound, "machine not found: %s", req.Id)
}
return nil, status.Errorf(codes.Internal, "delete machine from store: %v", err)
}
slog.Info("Machine removed from the cluster.", "id", req.Id)
return &emptypb.Empty{}, nil
}
-1
View File
@@ -4,7 +4,6 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"github.com/psviderski/uncloud/internal/dns" "github.com/psviderski/uncloud/internal/dns"
"github.com/psviderski/uncloud/internal/machine/api/pb" "github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/store" "github.com/psviderski/uncloud/internal/machine/store"
+1 -2
View File
@@ -3,9 +3,8 @@ package cluster
import ( import (
"errors" "errors"
"fmt" "fmt"
"net/netip"
"go4.org/netipx" "go4.org/netipx"
"net/netip"
) )
const DefaultSubnetBits = 24 const DefaultSubnetBits = 24
-1
View File
@@ -2,7 +2,6 @@ package cluster
import ( import (
"fmt" "fmt"
"github.com/psviderski/uncloud/internal/secret" "github.com/psviderski/uncloud/internal/secret"
) )
-6
View File
@@ -1,6 +0,0 @@
package constants
const (
// MachineAPIPort is the port for the Machine API service on the management WireGuard network.
MachineAPIPort = 51000
)
+4 -5
View File
@@ -3,11 +3,10 @@ package corroservice
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"github.com/BurntSushi/toml"
"net/netip" "net/netip"
"os" "os"
"path/filepath" "path/filepath"
"github.com/BurntSushi/toml"
"github.com/psviderski/uncloud/internal/fs" "github.com/psviderski/uncloud/internal/fs"
) )
@@ -51,7 +50,7 @@ func (c *Config) Write(path, owner string) error {
if err := encoder.Encode(c); err != nil { if err := encoder.Encode(c); err != nil {
return fmt.Errorf("encode config: %w", err) return fmt.Errorf("encode config: %w", err)
} }
if err := os.WriteFile(path, data.Bytes(), 0o600); err != nil { if err := os.WriteFile(path, data.Bytes(), 0600); err != nil {
return err return err
} }
if err := fs.Chown(path, owner, owner); err != nil { if err := fs.Chown(path, owner, owner); err != nil {
@@ -63,10 +62,10 @@ func (c *Config) Write(path, owner string) error {
func MkDataDir(dir, owner string) error { func MkDataDir(dir, owner string) error {
parent, _ := filepath.Split(dir) parent, _ := filepath.Split(dir)
// Use 0711 for parent directories to allow `owner` to access its nested data directory. // Use 0711 for parent directories to allow `owner` to access its nested data directory.
if err := os.MkdirAll(parent, 0o711); err != nil { if err := os.MkdirAll(parent, 0711); err != nil {
return fmt.Errorf("create directory %q: %w", parent, err) return fmt.Errorf("create directory %q: %w", parent, err)
} }
if err := os.Mkdir(dir, 0o700); err != nil { if err := os.Mkdir(dir, 0700); err != nil {
if !os.IsExist(err) { if !os.IsExist(err) {
return fmt.Errorf("create directory %q: %w", dir, err) return fmt.Errorf("create directory %q: %w", dir, err)
} }
+17 -36
View File
@@ -3,16 +3,15 @@ package corroservice
import ( import (
"context" "context"
"fmt" "fmt"
"io"
"log/slog"
"path/filepath"
"time"
"github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/network"
"github.com/docker/docker/client" "github.com/docker/docker/client"
"io"
"log/slog"
"path/filepath"
"time"
) )
const ( const (
@@ -27,47 +26,29 @@ type DockerService struct {
User string User string
} }
func NewDockerService(cli *client.Client, image, name, dataDir string) *DockerService {
return &DockerService{
Client: cli,
Image: image,
Name: name,
DataDir: dataDir,
}
}
func (s *DockerService) Start(ctx context.Context) error { func (s *DockerService) Start(ctx context.Context) error {
_, err := s.Client.ContainerInspect(ctx, s.Name) _, err := s.Client.ContainerInspect(ctx, s.Name)
if err != nil { if err != nil {
if !client.IsErrNotFound(err) { if client.IsErrNotFound(err) {
return s.startNewContainer(ctx)
}
return fmt.Errorf("inspect container %q: %w", s.Name, err) return fmt.Errorf("inspect container %q: %w", s.Name, err)
} }
if err = s.startNewContainer(ctx); err != nil {
return err
}
} else {
// Container already exists.
// TODO: recreate only if the container configuration has to be changed. // TODO: recreate only if the container configuration has to be changed.
if err = s.Client.ContainerRemove(ctx, s.Name, container.RemoveOptions{Force: true}); err != nil { if err = s.Client.ContainerRemove(ctx, s.Name, container.RemoveOptions{Force: true}); err != nil {
return fmt.Errorf("remove container %q: %w", s.Name, err) return fmt.Errorf("remove container %q: %w", s.Name, err)
} }
if err = s.startNewContainer(ctx); err != nil {
return err
}
}
slog.Debug("Waiting for corrosion service to be ready.") return s.startNewContainer(ctx)
if err = WaitReady(ctx, s.DataDir); err != nil {
return err
}
slog.Debug("Corrosion service is ready.")
return nil
}
func (s *DockerService) Stop(ctx context.Context) error {
if err := s.Client.ContainerStop(ctx, s.Name, container.StopOptions{}); err != nil {
return fmt.Errorf("stop container %q: %w", s.Name, err)
}
slog.Debug("Corrosion Docker container stopped.", "name", s.Name)
if err := s.Client.ContainerRemove(ctx, s.Name, container.RemoveOptions{}); err != nil {
return fmt.Errorf("remove container %q: %w", s.Name, err)
}
slog.Debug("Corrosion Docker container removed.", "name", s.Name)
return nil
} }
func (s *DockerService) Restart(ctx context.Context) error { func (s *DockerService) Restart(ctx context.Context) error {
+1 -54
View File
@@ -1,62 +1,9 @@
package corroservice package corroservice
import ( import "context"
"context"
"fmt"
"os"
"path/filepath"
"time"
"github.com/BurntSushi/toml"
"github.com/cenkalti/backoff/v4"
"github.com/psviderski/uncloud/internal/corrosion"
)
type Service interface { type Service interface {
Start(ctx context.Context) error Start(ctx context.Context) error
Stop(ctx context.Context) error
Restart(ctx context.Context) error Restart(ctx context.Context) error
Running() bool Running() bool
} }
// WaitReady waits for the Corrosion service to be ready with the uncloud schema applied.
func WaitReady(ctx context.Context, dataDir string) error {
// Read the config file to get the API address.
configPath := filepath.Join(dataDir, "config.toml")
configData, err := os.ReadFile(configPath)
if err != nil {
return fmt.Errorf("read config file: %w", err)
}
var config Config
if err = toml.Unmarshal(configData, &config); err != nil {
return fmt.Errorf("unmarshal config: %w", err)
}
corro, err := corrosion.NewAPIClient(config.API.Addr)
if err != nil {
return fmt.Errorf("create corrosion API client: %w", err)
}
// Corrosion starts serving the API before applying the schema. Query the cluster table with exponential backoff
// to check if the uncloud schema has been applied.
checkReady := func() error {
rows, err := corro.QueryContext(ctx, "SELECT 1 FROM cluster LIMIT 1")
if err != nil {
return fmt.Errorf("query cluster table: %w", err)
}
defer rows.Close()
return nil
}
b := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(50*time.Millisecond),
backoff.WithMaxInterval(1*time.Second),
backoff.WithMaxElapsedTime(15*time.Second),
)
if err = backoff.Retry(checkReady, backoff.WithContext(b, ctx)); err != nil {
return fmt.Errorf("corrosion service did not become ready: %w", err)
}
return nil
}
+157
View File
@@ -0,0 +1,157 @@
package corroservice
import (
"bufio"
"context"
"fmt"
"log/slog"
"os/exec"
"path/filepath"
"sync"
"syscall"
"time"
)
const (
DefaultCommand = "corrosion"
DefaultDataDir = "/var/lib/uncloud/corrosion"
)
// SubprocessService implements the Service interface by running the service as a subprocess.
type SubprocessService struct {
Command string
DataDir string
cmd *exec.Cmd
running bool
mu sync.Mutex
cancelWatch context.CancelFunc
}
func DefaultSubprocessService() *SubprocessService {
return &SubprocessService{
Command: DefaultCommand,
DataDir: DefaultDataDir,
}
}
// TODO: maybe stop the process if this ctx is cancelled.
func (s *SubprocessService) Start(ctx context.Context) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.running {
return nil
}
return s.startProcess(ctx)
}
func (s *SubprocessService) Restart(ctx context.Context) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.running {
if err := s.stopProcess(); err != nil {
return fmt.Errorf("stop process: %w", err)
}
}
return s.startProcess(ctx)
}
func (s *SubprocessService) Running() bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.running
}
func (s *SubprocessService) startProcess(ctx context.Context) error {
s.cmd = exec.Command(s.Command, "agent", "-c", filepath.Join(s.DataDir, "config.toml"))
// Redirect stdout and stderr to the logger.
stdout, err := s.cmd.StdoutPipe()
if err != nil {
return fmt.Errorf("create stdout pipe: %w", err)
}
stderr, err := s.cmd.StderrPipe()
if err != nil {
return fmt.Errorf("create stderr pipe: %w", err)
}
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
slog.Info("[corrosion]: " + scanner.Text())
}
// TODO: remove
slog.Info("######## corrosion redirect go routine end ########")
}()
go func() {
scanner := bufio.NewScanner(stderr)
for scanner.Scan() {
slog.Error("[corrosion]: " + scanner.Text())
}
}()
if err = s.cmd.Start(); err != nil {
return fmt.Errorf("start process: %w", err)
}
s.running = true
// Watch for process exit to update running status.
go func() {
if err := s.cmd.Wait(); err != nil {
slog.Error("corrosion process exited with error.", "code", s.cmd.ProcessState.ExitCode(), "err", err)
}
s.mu.Lock()
s.running = false
s.mu.Unlock()
}()
// TODO: figure out the waiting process
// Wait for initialization
//timer := time.NewTimer(2 * time.Second)
//defer timer.Stop()
//select {
////case <-timer.C:
//// s.running = true
//// return nil
//case <-watchCtx.Done():
// return fmt.Errorf("process failed to start")
//case <-ctx.Done():
// s.stopProcess()
// return ctx.Err()
//}
return nil
}
func (s *SubprocessService) stopProcess() error {
if s.cmd == nil || s.cmd.Process == nil {
return nil
}
if err := s.cmd.Process.Signal(syscall.SIGTERM); err != nil {
return fmt.Errorf("send SIGTERM: %w", err)
}
// Wait up to 5 seconds for graceful shutdown before killing the process.
done := make(chan error, 1)
go func() {
done <- s.cmd.Wait()
}()
select {
case <-time.After(5 * time.Second):
if err := s.cmd.Process.Kill(); err != nil {
return fmt.Errorf("kill process: %w", err)
}
case err := <-done:
if err != nil {
return fmt.Errorf("process exited with error: %w", err)
}
}
return nil
}
+12 -15
View File
@@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"log/slog" "log/slog"
"os/exec" "os/exec"
"time"
) )
const DefaultSystemdUnit = "uncloud-corrosion.service" const DefaultSystemdUnit = "uncloud-corrosion.service"
@@ -26,15 +27,6 @@ func (s *SystemdService) Start(ctx context.Context) error {
return s.startOrRestart(ctx, "start") return s.startOrRestart(ctx, "start")
} }
func (s *SystemdService) Stop(_ context.Context) error {
if _, err := exec.Command("systemctl", "stop", s.Unit).Output(); err != nil {
return fmt.Errorf("systemctl stop %s: %w", s.Unit, err)
}
slog.Info("Corrosion systemd service stopped.", "unit", s.Unit)
return nil
}
func (s *SystemdService) Restart(ctx context.Context) error { func (s *SystemdService) Restart(ctx context.Context) error {
return s.startOrRestart(ctx, "restart") return s.startOrRestart(ctx, "restart")
} }
@@ -43,15 +35,20 @@ func (s *SystemdService) startOrRestart(ctx context.Context, cmd string) error {
if _, err := exec.Command("systemctl", cmd, s.Unit).Output(); err != nil { if _, err := exec.Command("systemctl", cmd, s.Unit).Output(); err != nil {
return fmt.Errorf("systemctl %s %s: %w", cmd, s.Unit, err) return fmt.Errorf("systemctl %s %s: %w", cmd, s.Unit, err)
} }
slog.Debug(fmt.Sprintf("Corrosion systemd service %sed.", cmd), "unit", s.Unit) slog.Info(fmt.Sprintf("Corrosion systemd service %sed.", cmd), "unit", s.Unit)
slog.Debug("Waiting for corrosion service to be ready.") // Optimistically wait for the corrosion service to start and initialise the database schema before proceeding.
if err := WaitReady(ctx, s.DataDir); err != nil { timer := time.NewTimer(2 * time.Second)
return err defer timer.Stop()
select {
case <-timer.C:
case <-ctx.Done():
return nil
} }
slog.Debug("Corrosion service is ready.")
s.running = true
// TODO: run a goroutine to check the status of the service and log any errors in the uncloud log.
s.running = true
return nil return nil
} }
+2 -2
View File
@@ -14,13 +14,13 @@ const DBFileName = "machine.db"
func NewDB(path string) (*sqlx.DB, error) { func NewDB(path string) (*sqlx.DB, error) {
// Create the database file with 0600 permissions if it doesn't exist, or update permissions if exists. // Create the database file with 0600 permissions if it doesn't exist, or update permissions if exists.
if _, err := os.Stat(path); os.IsNotExist(err) { if _, err := os.Stat(path); os.IsNotExist(err) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
if err != nil { if err != nil {
return nil, fmt.Errorf("create SQLite database '%s': %w", path, err) return nil, fmt.Errorf("create SQLite database '%s': %w", path, err)
} }
file.Close() file.Close()
} else { } else {
if err = os.Chmod(path, 0o600); err != nil { if err = os.Chmod(path, 0600); err != nil {
return nil, fmt.Errorf("update SQLite database permissions '%s': %w", path, err) return nil, fmt.Errorf("update SQLite database permissions '%s': %w", path, err)
} }
} }
+8 -2
View File
@@ -5,10 +5,12 @@ import (
"fmt" "fmt"
"log/slog" "log/slog"
"net/netip" "net/netip"
"strings"
"sync" "sync"
"time" "time"
"github.com/psviderski/uncloud/internal/machine/store" "github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/pkg/api"
) )
// ClusterResolver implements Resolver by tracking containers in the cluster and resolving service names // ClusterResolver implements Resolver by tracking containers in the cluster and resolving service names
@@ -82,13 +84,17 @@ func (r *ClusterResolver) updateServiceIPs(containers []store.ContainerRecord) {
continue continue
} }
ctr := record.Container ctr := api.ServiceContainer{Container: record.Container}
if ctr.ServiceID() == "" || ctr.ServiceName() == "" { if ctr.ServiceID() == "" || ctr.ServiceName() == "" {
// Container is not part of a service, skip it. // Container is not part of a service, skip it.
continue continue
} }
newServiceIPs[ctr.ServiceName()] = append(newServiceIPs[ctr.ServiceName()], ip) // TODO: remove normalisation after implementing service name validation:
//.https://github.com/psviderski/uncloud/issues/53
serviceName := strings.ToLower(ctr.ServiceName())
newServiceIPs[serviceName] = append(newServiceIPs[serviceName], ip)
// Also add the service ID as a valid lookup. // Also add the service ID as a valid lookup.
newServiceIPs[ctr.ServiceID()] = append(newServiceIPs[ctr.ServiceID()], ip) newServiceIPs[ctr.ServiceID()] = append(newServiceIPs[ctr.ServiceID()], ip)
containersCount++ containersCount++
+2 -16
View File
@@ -196,27 +196,13 @@ func (c *Client) RemoveContainer(ctx context.Context, id string, opts container.
return err return err
} }
// PullOptions defines the options for pulling an image from a remote registry.
// This is a copy of image.PullOptions from the Docker API without the PrivilegeFunc field that is non-serialisable.
type PullOptions struct {
All bool
// RegistryAuth is the base64 encoded credentials for the registry.
RegistryAuth string
Platform string
}
type PullImageMessage struct { type PullImageMessage struct {
Message jsonmessage.JSONMessage Message jsonmessage.JSONMessage
Err error Err error
} }
func (c *Client) PullImage(ctx context.Context, image string, opts PullOptions) (<-chan PullImageMessage, error) { func (c *Client) PullImage(ctx context.Context, image string) (<-chan PullImageMessage, error) {
optsBytes, err := json.Marshal(opts) stream, err := c.grpcClient.PullImage(ctx, &pb.PullImageRequest{Image: image})
if err != nil {
return nil, fmt.Errorf("marshal options: %w", err)
}
stream, err := c.grpcClient.PullImage(ctx, &pb.PullImageRequest{Image: image, Options: optsBytes})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -1,19 +0,0 @@
//go:build darwin
package docker
import (
"context"
"fmt"
"net/netip"
)
// EnsureUncloudNetwork is a stub for Darwin.
func (c *Controller) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error {
return fmt.Errorf("not supported on Darwin")
}
// Cleanup is a stub for Darwin.
func (c *Controller) Cleanup() error {
return fmt.Errorf("not supported on Darwin")
}
-234
View File
@@ -1,234 +0,0 @@
package docker
import (
"context"
"errors"
"fmt"
"log/slog"
"net/netip"
"strconv"
dockercontainer "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
dnetwork "github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"github.com/docker/docker/libnetwork/iptables"
"github.com/psviderski/uncloud/internal/machine/dns"
"github.com/psviderski/uncloud/internal/machine/firewall"
"github.com/psviderski/uncloud/internal/machine/network"
"github.com/psviderski/uncloud/pkg/api"
)
// EnsureUncloudNetwork creates the Docker bridge network NetworkName with the provided machine subnet
// if it doesn't exist. If the network exists but has a different subnet, it removes and recreates the network.
// It also configures iptables to allow container access from the WireGuard network.
func (c *Controller) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error {
// Ensure the Docker network 'uncloud' is created with the correct subnet.
needsCreation := false
nw, err := c.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{})
if err != nil {
if !client.IsErrNotFound(err) {
return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
}
needsCreation = true
} else if nw.IPAM.Config[0].Subnet != subnet.String() {
// Remove the Docker network if the subnet is different.
// It could be a leftover from a previous incomplete cleanup.
slog.Info(
"Removing Docker network with old subnet.", "name", NetworkName, "subnet", nw.IPAM.Config[0].Subnet,
)
if err = c.client.NetworkRemove(ctx, NetworkName); err != nil {
// It can still fail if the network is in use by a container. Leave it to the user to resolve the issue.
return fmt.Errorf("remove Docker network '%s': %w", NetworkName, err)
}
needsCreation = true
}
if needsCreation {
if _, err = c.client.NetworkCreate(
ctx, NetworkName, dnetwork.CreateOptions{
Driver: "bridge",
Scope: "local",
IPAM: &dnetwork.IPAM{
Config: []dnetwork.IPAMConfig{
{
Subnet: subnet.String(),
},
},
},
Labels: map[string]string{
api.LabelManaged: "",
},
Options: map[string]string{
// Starting with Docker 28.2.0 (https://github.com/moby/moby/pull/49832), we have to explicitly
// allow direct routing from the WireGuard interface to the bridge network.
"com.docker.network.bridge.trusted_host_interfaces": network.WireGuardInterfaceName,
},
},
); err != nil {
return fmt.Errorf("create Docker network '%s': %w", NetworkName, err)
}
slog.Info("Docker network created.", "name", NetworkName, "subnet", subnet.String())
if nw, err = c.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{}); err != nil {
return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
}
}
// Configure iptables to allow WireGuard network to access containers. The Docker daemon should have already
// created the DOCKER-USER chain at this point.
// TODO: check if this works when firewalld used instead of raw iptables. The Docker daemon has a different
// code path for firewalld.
// Bridge name doesn't seem to be documented but this is the source code where it is generated:
// https://github.com/moby/moby/blob/v27.2.1/libnetwork/drivers/bridge/bridge_linux.go#L664
bridgeName := "br-" + nw.ID[:12]
if err = configureIptables(bridgeName, subnet, dnsServer); err != nil {
return fmt.Errorf("configure iptables for Docker network '%s': %w", NetworkName, err)
}
return nil
}
// configureIptables configures iptables rules for the uncloud Docker network.
func configureIptables(bridgeName string, subnet netip.Prefix, dnsServer netip.Addr) error {
ipt := iptables.GetIptable(iptables.IPv4)
// Allow traffic from other machines and their containers through the WG mesh to the Uncloud containers
// on the machine.
wgRule := []string{
"--in-interface", network.WireGuardInterfaceName,
"--out-interface", bridgeName,
"-j", "ACCEPT",
}
if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Insert, wgRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
// Allow DNS queries from Uncloud containers to the embedded DNS server.
for _, proto := range []string{"udp", "tcp"} {
dnsRule := []string{
"--in-interface", bridgeName,
"--dst", dnsServer.String(),
"--protocol", proto,
"--dport", strconv.Itoa(dns.Port),
"-j", "ACCEPT",
}
if err := ipt.ProgramRule(iptables.Filter, firewall.UncloudInputChain, iptables.Insert, dnsRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
}
// Skip masquerading for the container traffic going from the uncloud Docker network through the WG mesh.
// https://uncloud.run/blog/connect-docker-containers-across-hosts-wireguard#step-3-configure-ip-routing
skipMasqueradeRule := []string{
"--src", subnet.String(),
"--out-interface", network.WireGuardInterfaceName,
"-j", "RETURN",
}
// Delete and reinsert the rule to ensure it's at the top of the POSTROUTING chain before the MASQUERADE rule
// added by Docker: POSTROUTING -s 10.210.X.0/24 ! -o br-XXX -j MASQUERADE
if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil {
return fmt.Errorf("delete iptables rule: %w", err)
}
if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Insert, skipMasqueradeRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
return nil
}
// cleanupIptables deletes the iptables rules for the uncloud Docker network.
func cleanupIptables(bridgeName string, subnet netip.Prefix) error {
ipt := iptables.GetIptable(iptables.IPv4)
// Delete the rule allowing traffic from the WireGuard network to the Docker bridge.
wgRule := []string{
"--in-interface", network.WireGuardInterfaceName,
"--out-interface", bridgeName,
"-j", "ACCEPT",
}
if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Delete, wgRule); err != nil {
return fmt.Errorf("delete iptables rule: %w", err)
}
// Delete the rule that skips masquerading for the container traffic going from the uncloud Docker network
// through the WG mesh.
skipMasqueradeRule := []string{
"--src", subnet.String(),
"--out-interface", network.WireGuardInterfaceName,
"-j", "RETURN",
}
if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil {
return fmt.Errorf("delete iptables rule: %w", err)
}
// Rules in uncloud-owned chains will be automatically cleaned up by the machine cleanup.
return nil
}
// Cleanup removes all uncloud-managed containers and the uncloud Docker network.
func (c *Controller) Cleanup() error {
ctx := context.Background()
var errs []error
// Remove uncloud-managed Docker containers.
containers, err := c.client.ContainerList(ctx, dockercontainer.ListOptions{
All: true, // Include stopped containers.
Filters: filters.NewArgs(
filters.Arg("label", api.LabelManaged),
),
})
if err != nil {
errs = append(errs, fmt.Errorf("list uncloud-managed Docker containers: %w", err))
} else if len(containers) > 0 {
slog.Info("Removing uncloud-managed Docker containers.", "count", len(containers))
removed := 0
for _, ctr := range containers {
err = c.client.ContainerStop(ctx, ctr.ID, dockercontainer.StopOptions{})
if err != nil && !client.IsErrNotFound(err) {
errs = append(errs, fmt.Errorf("stop container '%s': %w", ctr.ID, err))
}
err = c.client.ContainerRemove(ctx, ctr.ID, dockercontainer.RemoveOptions{
// Remove anonymous volumes created by the container.
RemoveVolumes: true,
})
if err == nil {
removed++
} else if !client.IsErrNotFound(err) {
errs = append(errs, fmt.Errorf("remove container '%s': %w", ctr.ID, err))
}
}
slog.Info("Removed uncloud-managed Docker containers.", "count", removed)
}
// Remove the uncloud Docker network and related iptables rules.
nw, err := c.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{})
if err == nil {
bridgeName := "br-" + nw.ID[:12]
var subnet netip.Prefix
if len(nw.IPAM.Config) > 0 {
subnet, _ = netip.ParsePrefix(nw.IPAM.Config[0].Subnet)
}
if subnet.IsValid() {
if err = cleanupIptables(bridgeName, subnet); err != nil {
errs = append(errs, fmt.Errorf("cleanup iptables for Docker network '%s': %w", NetworkName, err))
} else {
slog.Info("Cleaned up iptables rules for Docker network.", "name", NetworkName, "bridge", bridgeName)
}
}
if err = c.client.NetworkRemove(ctx, NetworkName); err == nil {
slog.Info("Docker network removed.", "name", NetworkName)
} else if !client.IsErrNotFound(err) {
errs = append(errs, fmt.Errorf("remove Docker network '%s': %w", NetworkName, err))
}
} else if !client.IsErrNotFound(err) {
errs = append(errs, fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err))
}
return errors.Join(errs...)
}
@@ -7,11 +7,12 @@ import (
"log/slog" "log/slog"
"time" "time"
"github.com/docker/docker/api/types/container" dockercontainer "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/events" "github.com/docker/docker/api/types/events"
"github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client" "github.com/docker/docker/client"
"github.com/psviderski/uncloud/internal/machine/store" "github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/pkg/api"
) )
const ( const (
@@ -23,26 +24,23 @@ const (
SyncInterval = 30 * time.Second SyncInterval = 30 * time.Second
) )
// Controller monitors Docker events and synchronises service containers with the cluster store. type Manager struct {
type Controller struct { client *client.Client
// machineID is the ID of the machine where the managed Docker daemon is running. // machineID is the ID of the machine where the managed Docker daemon is running.
machineID string machineID string
client *client.Client
service *Service
store *store.Store store *store.Store
} }
func NewController(machineID string, service *Service, store *store.Store) *Controller { func NewManager(client *client.Client, machineID string, store *store.Store) *Manager {
return &Controller{ return &Manager{
client: client,
machineID: machineID, machineID: machineID,
client: service.Client,
service: service,
store: store, store: store,
} }
} }
// WaitDaemonReady waits for the Docker daemon to start and be ready to serve requests. // WaitDaemonReady waits for the Docker daemon to start and be ready to serve requests.
func (c *Controller) WaitDaemonReady(ctx context.Context) error { func (m *Manager) WaitDaemonReady(ctx context.Context) error {
ticker := time.NewTicker(1 * time.Second) ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop() defer ticker.Stop()
@@ -52,7 +50,7 @@ func (c *Controller) WaitDaemonReady(ctx context.Context) error {
case <-ctx.Done(): case <-ctx.Done():
return ctx.Err() return ctx.Err()
case <-ticker.C: case <-ticker.C:
_, err := c.client.Ping(ctx) _, err := m.client.Ping(ctx)
if err == nil { if err == nil {
ready = true ready = true
break break
@@ -69,7 +67,7 @@ func (c *Controller) WaitDaemonReady(ctx context.Context) error {
return nil return nil
} }
func (c *Controller) WatchAndSyncContainers(ctx context.Context) error { func (m *Manager) WatchAndSyncContainers(ctx context.Context) error {
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
// Filter only local container events. // Filter only local container events.
@@ -81,9 +79,9 @@ func (c *Controller) WatchAndSyncContainers(ctx context.Context) error {
} }
// Subscribe to Docker events before running the initial sync to avoid missing any events. // Subscribe to Docker events before running the initial sync to avoid missing any events.
eventCh, errCh := c.service.Client.Events(ctx, opts) eventCh, errCh := m.client.Events(ctx, opts)
slog.Debug("Syncing containers to cluster store before processing Docker events.") slog.Debug("Syncing containers to cluster store before processing Docker events.")
if err := c.syncContainersToStore(ctx); err != nil { if err := m.syncContainersToStore(ctx); err != nil {
// The deferred cancel will stop the event subscription. // The deferred cancel will stop the event subscription.
return fmt.Errorf("sync containers to cluster store: %w", err) return fmt.Errorf("sync containers to cluster store: %w", err)
} }
@@ -128,13 +126,13 @@ func (c *Controller) WatchAndSyncContainers(ctx context.Context) error {
"container_name", e.Actor.Attributes["name"], "container_name", e.Actor.Attributes["name"],
"action", e.Action) "action", e.Action)
if err := c.syncContainersToStore(ctx); err != nil { if err := m.syncContainersToStore(ctx); err != nil {
return fmt.Errorf("sync containers to cluster store: %w", err) return fmt.Errorf("sync containers to cluster store: %w", err)
} }
case <-ticker.C: case <-ticker.C:
slog.Debug("Syncing containers to cluster store triggered by a regular interval.", slog.Debug("Syncing containers to cluster store triggered by a regular interval.",
"interval", SyncInterval) "interval", SyncInterval)
if err := c.syncContainersToStore(ctx); err != nil { if err := m.syncContainersToStore(ctx); err != nil {
return fmt.Errorf("sync containers to cluster store: %w", err) return fmt.Errorf("sync containers to cluster store: %w", err)
} }
case err := <-errCh: case err := <-errCh:
@@ -146,16 +144,32 @@ func (c *Controller) WatchAndSyncContainers(ctx context.Context) error {
} }
} }
func (c *Controller) syncContainersToStore(ctx context.Context) error { func (m *Manager) syncContainersToStore(ctx context.Context) error {
storeContainers, err := c.store.ListContainers(ctx, store.ListOptions{MachineIDs: []string{c.machineID}}) storeContainers, err := m.store.ListContainers(ctx, store.ListOptions{MachineIDs: []string{m.machineID}})
if err != nil { if err != nil {
return fmt.Errorf("list containers from store: %w", err) return fmt.Errorf("list containers from store: %w", err)
} }
containers, err := c.service.ListServiceContainers(ctx, "", container.ListOptions{}) // List only Uncloud service containers identified by their labels.
containerSummaries, err := m.client.ContainerList(ctx, dockercontainer.ListOptions{
Filters: filters.NewArgs(
filters.Arg("label", api.LabelServiceID),
filters.Arg("label", api.LabelServiceName),
),
})
if err != nil { if err != nil {
// TODO: mark all containers as outdated in the store. // TODO: mark all containers as outdated in the store.
return fmt.Errorf("list service containers: %w", err) return fmt.Errorf("list Docker containers: %w", err)
}
// Inspect each container to get the full container details.
containers := make([]api.Container, len(containerSummaries))
for i, cs := range containerSummaries {
ctr, err := m.client.ContainerInspect(ctx, cs.ID)
if err != nil {
return fmt.Errorf("inspect container '%s': %w", cs.ID, err)
}
containers[i] = api.Container{ContainerJSON: ctr}
} }
// Delete containers from the store that are no longer present in the Docker daemon. // Delete containers from the store that are no longer present in the Docker daemon.
@@ -175,15 +189,15 @@ func (c *Controller) syncContainersToStore(ctx context.Context) error {
var storeErr error var storeErr error
if len(deleteIDs) > 0 { if len(deleteIDs) > 0 {
if err = c.store.DeleteContainers(ctx, store.DeleteOptions{IDs: deleteIDs}); err != nil { if err = m.store.DeleteContainers(ctx, store.DeleteOptions{IDs: deleteIDs}); err != nil {
storeErr = fmt.Errorf("delete containers from store: %w", err) storeErr = fmt.Errorf("delete containers from store: %w", err)
} }
} }
// Create or update the current Docker containers in the store. // Create or update the current Docker containers in the store.
for _, ctr := range containers { for _, c := range containers {
if err = c.store.CreateOrUpdateContainer(ctx, ctr, c.machineID); err != nil { if err = m.store.CreateOrUpdateContainer(ctx, c, m.machineID); err != nil {
storeErr = errors.Join(storeErr, fmt.Errorf("create or update container '%s': %w", ctr.ID, err)) storeErr = errors.Join(storeErr, fmt.Errorf("create or update container %q: %w", c.ID, err))
} }
} }
return storeErr return storeErr
+14
View File
@@ -0,0 +1,14 @@
//go:build darwin
package docker
import (
"context"
"fmt"
"net/netip"
)
// EnsureUncloudNetwork is a stub for Darwin.
func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error {
return fmt.Errorf("not supported on Darwin")
}
+107
View File
@@ -0,0 +1,107 @@
package docker
import (
"context"
"fmt"
"log/slog"
"net/netip"
"strconv"
dnetwork "github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"github.com/docker/docker/libnetwork/iptables"
"github.com/psviderski/uncloud/internal/machine/dns"
"github.com/psviderski/uncloud/internal/machine/firewall"
"github.com/psviderski/uncloud/internal/machine/network"
)
// EnsureUncloudNetwork creates the Docker bridge network NetworkName with the provided machine subnet
// if it doesn't exist. If the network exists but has a different subnet, it removes and recreates the network.
// It also configures iptables to allow container access from the WireGuard network.
func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error {
// Ensure the Docker network 'uncloud' is created with the correct subnet.
needsCreation := false
nw, err := m.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{})
if err != nil {
if !client.IsErrNotFound(err) {
return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
}
needsCreation = true
} else if nw.IPAM.Config[0].Subnet != subnet.String() {
// Remove the Docker network if the subnet is different.
// It could be a leftover from a previous incomplete cleanup.
slog.Info(
"Removing Docker network with old subnet.", "name", NetworkName, "subnet", nw.IPAM.Config[0].Subnet,
)
if err = m.client.NetworkRemove(ctx, NetworkName); err != nil {
// It can still fail if the network is in use by a container. Leave it to the user to resolve the issue.
return fmt.Errorf("remove Docker network '%s': %w", NetworkName, err)
}
needsCreation = true
}
if needsCreation {
if _, err = m.client.NetworkCreate(
ctx, NetworkName, dnetwork.CreateOptions{
Driver: "bridge",
Scope: "local",
IPAM: &dnetwork.IPAM{
Config: []dnetwork.IPAMConfig{
{
Subnet: subnet.String(),
},
},
},
},
); err != nil {
return fmt.Errorf("create Docker network '%s': %w", NetworkName, err)
}
slog.Info("Docker network created.", "name", NetworkName, "subnet", subnet.String())
if nw, err = m.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{}); err != nil {
return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
}
}
// Configure iptables to allow WireGuard network to access containers. The Docker daemon should have already
// created the DOCKER-USER chain at this point.
// TODO: check if this works when firewalld used instead of raw iptables. The Docker daemon has a different
// code path for firewalld.
// Bridge name doesn't seem to be documented but this is the source code where it is generated:
// https://github.com/moby/moby/blob/v27.2.1/libnetwork/drivers/bridge/bridge_linux.go#L664
bridgeName := "br-" + nw.ID[:12]
if err = configureIptables(bridgeName, dnsServer); err != nil {
return fmt.Errorf("configure iptables for Docker network '%s': %w", NetworkName, err)
}
return nil
}
// configureIptables configures iptables rules for the uncloud Docker network.
func configureIptables(bridgeName string, dnsServer netip.Addr) error {
ipt := iptables.GetIptable(iptables.IPv4)
// Allow traffic from other machines and their containers through the WG mesh to the Uncloud containers
// on the machine.
wgRule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"}
if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Insert, wgRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
// Allow DNS queries from Uncloud containers to the embedded DNS server.
for _, proto := range []string{"udp", "tcp"} {
dnsRule := []string{
"--in-interface", bridgeName,
"--dst", dnsServer.String(),
"--protocol", proto,
"--dport", strconv.Itoa(dns.Port),
"-j", "ACCEPT",
}
if err := ipt.ProgramRule(iptables.Filter, firewall.UncloudInputChain, iptables.Insert, dnsRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
}
return nil
}
+53 -70
View File
@@ -2,21 +2,19 @@ package docker
import ( import (
"context" "context"
"database/sql"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"log/slog" "log/slog"
"net/netip" "net/netip"
"os"
"regexp" "regexp"
"slices" "slices"
"strconv" "strconv"
"strings" "strings"
"github.com/distribution/reference" "github.com/distribution/reference"
dockercommand "github.com/docker/cli/cli/command"
dockerconfig "github.com/docker/cli/cli/config"
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/filters"
@@ -48,48 +46,19 @@ var fullDockerIDRegex = regexp.MustCompile(`^[a-f0-9]{64}$`)
type Server struct { type Server struct {
pb.UnimplementedDockerServer pb.UnimplementedDockerServer
client *client.Client client *client.Client
service *Service
db *sqlx.DB db *sqlx.DB
// internalDNSIP is a function that returns the IP address of the internal DNS server. It may return an empty // internalDNSIP is a function that returns the IP address of the internal DNS server. It may return an empty
// address if the address is unknown (e.g. when the machine is not initialised yet). // address if the address is unknown (e.g. when the machine is not initialised yet).
internalDNSIP func() netip.Addr internalDNSIP func() netip.Addr
// networkReady is a function that returns true if the Docker network is ready for containers.
networkReady func() bool
// waitForNetworkReady is a function that waits for the Docker network to be ready for containers.
waitForNetworkReady func(ctx context.Context) error
} }
// ServerOption configures the Docker server. // NewServer creates a new Docker gRPC server with the provided Docker client.
type ServerOption func(*Server) func NewServer(cli *client.Client, db *sqlx.DB, internalDNSIP func() netip.Addr) *Server {
return &Server{
// WithNetworkReady sets the network readiness check function. client: cli,
func WithNetworkReady(networkReady func() bool) ServerOption {
return func(s *Server) {
s.networkReady = networkReady
}
}
// WithWaitForNetworkReady sets the network readiness wait function.
func WithWaitForNetworkReady(waitForNetworkReady func(ctx context.Context) error) ServerOption {
return func(s *Server) {
s.waitForNetworkReady = waitForNetworkReady
}
}
// NewServer creates a new Docker gRPC server with the provided Docker service.
func NewServer(service *Service, db *sqlx.DB, internalDNSIP func() netip.Addr, opts ...ServerOption) *Server {
s := &Server{
client: service.Client,
service: service,
db: db, db: db,
internalDNSIP: internalDNSIP, internalDNSIP: internalDNSIP,
} }
for _, opt := range opts {
opt(s)
}
return s
} }
// CreateContainer creates a new container based on the given configuration. // CreateContainer creates a new container based on the given configuration.
@@ -149,15 +118,6 @@ func (s *Server) InspectContainer(ctx context.Context, req *pb.InspectContainerR
// StartContainer starts a container with the given ID and options. // StartContainer starts a container with the given ID and options.
func (s *Server) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (*emptypb.Empty, error) { func (s *Server) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (*emptypb.Empty, error) {
// Wait for Docker network to be ready before starting the container
if s.waitForNetworkReady != nil {
if err := s.waitForNetworkReady(ctx); err != nil {
return nil, status.Errorf(codes.Unavailable, "Docker network not ready: %v", err)
}
} else if s.networkReady != nil && !s.networkReady() {
return nil, status.Errorf(codes.Unavailable, "Docker network not ready")
}
var opts container.StartOptions var opts container.StartOptions
if len(req.Options) > 0 { if len(req.Options) > 0 {
if err := json.Unmarshal(req.Options, &opts); err != nil { if err := json.Unmarshal(req.Options, &opts); err != nil {
@@ -269,6 +229,7 @@ func (s *Server) RemoveContainer(ctx context.Context, req *pb.RemoveContainerReq
func (s *Server) PullImage(req *pb.PullImageRequest, stream grpc.ServerStreamingServer[pb.JSONMessage]) error { func (s *Server) PullImage(req *pb.PullImageRequest, stream grpc.ServerStreamingServer[pb.JSONMessage]) error {
ctx := stream.Context() ctx := stream.Context()
// TODO: replace with another JSON serializable type (PullOptions.PrivilegeFunc is not serializable).
var opts image.PullOptions var opts image.PullOptions
if len(req.Options) > 0 { if len(req.Options) > 0 {
if err := json.Unmarshal(req.Options, &opts); err != nil { if err := json.Unmarshal(req.Options, &opts); err != nil {
@@ -276,14 +237,6 @@ func (s *Server) PullImage(req *pb.PullImageRequest, stream grpc.ServerStreaming
} }
} }
if opts.RegistryAuth == "" {
// Try to retrieve the authentication token for the image from the default local Docker config file.
dockerConfig := dockerconfig.LoadDefaultConfigFile(os.Stderr)
if encodedAuth, err := dockercommand.RetrieveAuthTokenFromImage(dockerConfig, req.Image); err == nil {
opts.RegistryAuth = encodedAuth
}
}
respBody, err := s.client.ImagePull(ctx, req.Image, opts) respBody, err := s.client.ImagePull(ctx, req.Image, opts)
if err != nil { if err != nil {
return status.Errorf(codes.Internal, err.Error()) return status.Errorf(codes.Internal, err.Error())
@@ -466,7 +419,6 @@ func (s *Server) RemoveVolume(ctx context.Context, req *pb.RemoveVolumeRequest)
} }
// CreateServiceContainer creates a new container for the service with the given specifications. // CreateServiceContainer creates a new container for the service with the given specifications.
// TODO: move the main logic to the Docker service and remove db dependency from the server.
func (s *Server) CreateServiceContainer( func (s *Server) CreateServiceContainer(
ctx context.Context, req *pb.CreateServiceContainerRequest, ctx context.Context, req *pb.CreateServiceContainerRequest,
) (*pb.CreateContainerResponse, error) { ) (*pb.CreateContainerResponse, error) {
@@ -558,10 +510,10 @@ func (s *Server) CreateServiceContainer(
Memory: spec.Container.Resources.Memory, Memory: spec.Container.Resources.Memory,
MemoryReservation: spec.Container.Resources.MemoryReservation, MemoryReservation: spec.Container.Resources.MemoryReservation,
}, },
// Restart service containers if they exit or a machine restarts unless they are explicitly stopped. // Always restart service containers if they exit or a machine restarts.
// For one-off containers and batch jobs we plan to use a different service type/mode. // For one-off containers and batch jobs we plan to use a different service type/mode.
RestartPolicy: container.RestartPolicy{ RestartPolicy: container.RestartPolicy{
Name: container.RestartPolicyUnlessStopped, Name: container.RestartPolicyAlways,
}, },
} }
@@ -721,7 +673,7 @@ func (s *Server) verifyDockerVolumesExist(ctx context.Context, mounts []mount.Mo
func (s *Server) InspectServiceContainer( func (s *Server) InspectServiceContainer(
ctx context.Context, req *pb.InspectContainerRequest, ctx context.Context, req *pb.InspectContainerRequest,
) (*pb.ServiceContainer, error) { ) (*pb.ServiceContainer, error) {
serviceCtr, err := s.service.InspectServiceContainer(ctx, req.Id) ctr, err := s.client.ContainerInspect(ctx, req.Id)
if err != nil { if err != nil {
if client.IsErrNotFound(err) { if client.IsErrNotFound(err) {
return nil, status.Errorf(codes.NotFound, err.Error()) return nil, status.Errorf(codes.NotFound, err.Error())
@@ -729,14 +681,19 @@ func (s *Server) InspectServiceContainer(
return nil, status.Errorf(codes.Internal, err.Error()) return nil, status.Errorf(codes.Internal, err.Error())
} }
ctrBytes, err := json.Marshal(serviceCtr.Container) ctrBytes, err := json.Marshal(ctr)
if err != nil { if err != nil {
return nil, status.Errorf(codes.Internal, "marshal container: %v", err) return nil, status.Errorf(codes.Internal, "marshal response: %v", err)
} }
specBytes, err := json.Marshal(serviceCtr.ServiceSpec) var specBytes []byte
err = s.db.QueryRowContext(ctx, `SELECT service_spec FROM containers WHERE id = $1`, ctr.ID).Scan(&specBytes)
if err != nil { if err != nil {
return nil, status.Errorf(codes.Internal, "marshal service spec: %v", err) if errors.Is(err, sql.ErrNoRows) {
return nil, status.Errorf(codes.NotFound, "service spec not found for container: '%s'", ctr.ID)
}
return nil, status.Errorf(codes.Internal, "get service spec for container '%s' from machine database: %v",
ctr.ID, err)
} }
return &pb.ServiceContainer{ return &pb.ServiceContainer{
@@ -768,28 +725,54 @@ func (s *Server) ListServiceContainers(
return nil, status.Errorf(codes.InvalidArgument, "unmarshal filters: %v", err) return nil, status.Errorf(codes.InvalidArgument, "unmarshal filters: %v", err)
} }
opts.Filters = args opts.Filters = args
} else {
opts.Filters = filters.NewArgs()
} }
} }
// Only uncloud-managed containers that belong to some service.
opts.Filters.Add("label", api.LabelServiceID)
opts.Filters.Add("label", api.LabelManaged)
containers, err := s.service.ListServiceContainers(ctx, req.ServiceId, opts) containerSummaries, err := s.client.ContainerList(ctx, opts)
if err != nil { if err != nil {
return nil, status.Error(codes.Internal, err.Error()) return nil, status.Error(codes.Internal, err.Error())
} }
// Convert to protobuf format. containers := make([]*pb.ServiceContainer, 0, len(containerSummaries))
pbContainers := make([]*pb.ServiceContainer, 0, len(containers)) for _, cs := range containerSummaries {
for _, ctr := range containers { if req.ServiceId != "" &&
ctrBytes, err := json.Marshal(ctr.Container) cs.Labels[api.LabelServiceID] != req.ServiceId && cs.Labels[api.LabelServiceName] != req.ServiceId {
continue
}
ctr, err := s.client.ContainerInspect(ctx, cs.ID)
if err != nil {
if client.IsErrNotFound(err) {
// The listed container may have been removed while we were inspecting other containers.
continue
}
return nil, status.Errorf(codes.Internal, "inspect container %s: %v", cs.ID, err)
}
ctrBytes, err := json.Marshal(ctr)
if err != nil { if err != nil {
return nil, status.Errorf(codes.Internal, "marshal container: %v", err) return nil, status.Errorf(codes.Internal, "marshal container: %v", err)
} }
specBytes, err := json.Marshal(ctr.ServiceSpec) var specBytes []byte
err = s.db.QueryRowContext(ctx, `SELECT service_spec FROM containers WHERE id = $1`, ctr.ID).Scan(&specBytes)
if err != nil { if err != nil {
return nil, status.Errorf(codes.Internal, "marshal service spec: %v", err) if errors.Is(err, sql.ErrNoRows) {
// If this happens, there is a bug in the code, or someone manually removed the container from the DB,
// or created a managed container out of band.
slog.Error("Service container not found in machine database.", "id", ctr.ID)
// Just ignore such a container to not fail the list operation as it's not easily recoverable.
continue
}
return nil, status.Errorf(codes.Internal, "get service spec for container '%s' from machine database: %v",
ctr.ID, err)
} }
pbContainers = append(pbContainers, &pb.ServiceContainer{ containers = append(containers, &pb.ServiceContainer{
Container: ctrBytes, Container: ctrBytes,
ServiceSpec: specBytes, ServiceSpec: specBytes,
}) })
@@ -798,7 +781,7 @@ func (s *Server) ListServiceContainers(
return &pb.ListServiceContainersResponse{ return &pb.ListServiceContainersResponse{
Messages: []*pb.MachineServiceContainers{ Messages: []*pb.MachineServiceContainers{
{ {
Containers: pbContainers, Containers: containers,
}, },
}, },
}, nil }, nil
-103
View File
@@ -1,103 +0,0 @@
package docker
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"log/slog"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/jmoiron/sqlx"
"github.com/psviderski/uncloud/pkg/api"
)
// Service provides higher-level Docker operations that extends Docker API with Uncloud-specific data
// from the machine database.
type Service struct {
Client *client.Client
db *sqlx.DB
}
// NewService creates a new Docker service instance.
func NewService(client *client.Client, db *sqlx.DB) *Service {
return &Service{
Client: client,
db: db,
}
}
// InspectServiceContainer inspects a Docker container and retrieves its associated ServiceSpec
// from the machine database, returning a complete ServiceContainer.
func (s *Service) InspectServiceContainer(ctx context.Context, nameOrID string) (api.ServiceContainer, error) {
var serviceCtr api.ServiceContainer
ctr, err := s.Client.ContainerInspect(ctx, nameOrID)
if err != nil {
return serviceCtr, err
}
if _, ok := ctr.Config.Labels[api.LabelManaged]; !ok {
return serviceCtr, fmt.Errorf("container '%s' is not managed by Uncloud", nameOrID)
}
serviceCtr.Container = api.Container{ContainerJSON: ctr}
// Retrieve ServiceSpec from the machine database.
var specBytes []byte
err = s.db.QueryRowContext(ctx, `SELECT service_spec FROM containers WHERE id = $1`, ctr.ID).Scan(&specBytes)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
// If this happens, there is a bug in the code, or someone manually removed the container from the DB,
// or created a managed container out of band or by previous uncloud installation.
return serviceCtr, fmt.Errorf("service spec not found for container '%s' in machine DB", ctr.ID)
}
return serviceCtr, fmt.Errorf("get service spec for container '%s' from machine DB: %w", ctr.ID, err)
}
if err = json.Unmarshal(specBytes, &serviceCtr.ServiceSpec); err != nil {
return serviceCtr, fmt.Errorf("unmarshal service spec for container '%s': %w", ctr.ID, err)
}
return serviceCtr, nil
}
// ListServiceContainers lists Docker containers that belong to the service with the given name or ID.
// If serviceIDOrName is empty, all service containers are returned. The opts parameter allows additional filtering.
func (s *Service) ListServiceContainers(
ctx context.Context, serviceNameOrID string, opts container.ListOptions,
) ([]api.ServiceContainer, error) {
if opts.Filters.Len() == 0 {
opts.Filters = filters.NewArgs()
}
// Add labels to existing filters to list only Uncloud-managed service containers.
opts.Filters.Add("label", api.LabelServiceID)
opts.Filters.Add("label", api.LabelManaged)
containerSummaries, err := s.Client.ContainerList(ctx, opts)
if err != nil {
return nil, err
}
var containers []api.ServiceContainer
for _, cs := range containerSummaries {
// Filter by service name or ID if provided.
if serviceNameOrID != "" &&
cs.Labels[api.LabelServiceID] != serviceNameOrID &&
cs.Labels[api.LabelServiceName] != serviceNameOrID {
continue
}
ctr, err := s.InspectServiceContainer(ctx, cs.ID)
if err != nil {
// Log error but continue with other containers.
slog.Error("Failed to inspect service container.", "service", serviceNameOrID, "id", cs.ID, "err", err)
continue
}
containers = append(containers, ctr)
}
return containers, nil
}
@@ -6,8 +6,3 @@ import "fmt"
func ConfigureIptablesChains() error { func ConfigureIptablesChains() error {
return fmt.Errorf("not supported on Darwin") return fmt.Errorf("not supported on Darwin")
} }
// CleanupIptablesChains is a stub for Darwin.
func CleanupIptablesChains() error {
return fmt.Errorf("not supported on Darwin")
}
+18 -107
View File
@@ -2,13 +2,10 @@ package firewall
import ( import (
"fmt" "fmt"
"log/slog"
"strconv" "strconv"
"strings" "strings"
"github.com/docker/docker/libnetwork/iptables" "github.com/docker/docker/libnetwork/iptables"
"github.com/psviderski/uncloud/internal/machine/constants"
"github.com/psviderski/uncloud/internal/machine/corroservice"
"github.com/psviderski/uncloud/internal/machine/network" "github.com/psviderski/uncloud/internal/machine/network"
) )
@@ -19,72 +16,25 @@ const (
// ConfigureIptablesChains sets up custom iptables chains and initial firewall rules for Uncloud networking. // ConfigureIptablesChains sets up custom iptables chains and initial firewall rules for Uncloud networking.
func ConfigureIptablesChains() error { func ConfigureIptablesChains() error {
if err := createIptablesChains(); err != nil { // Ensure iptables UNCLOUD-INPUT chain with a RETURN rule exists. All existing rules are flushed.
return err ipt := iptables.GetIptable(iptables.IPv4)
}
ipt4 := iptables.GetIptable(iptables.IPv4)
ipt6 := iptables.GetIptable(iptables.IPv6)
// Allow WireGuard traffic to the machine.
acceptWireGuardRule := []string{"-p", "udp", "--dport", strconv.Itoa(network.WireGuardPort), "-j", "ACCEPT"}
err := ipt4.ProgramRule(iptables.Filter, UncloudInputChain, iptables.Insert, acceptWireGuardRule)
if err != nil {
return fmt.Errorf("insert iptables rule '%s': %w", strings.Join(acceptWireGuardRule, " "), err)
}
// Allow cluster machines to access Machine API via the management IPv6 WireGuard network.
acceptMachineAPIRule := []string{
"-i", network.WireGuardInterfaceName,
"-s", "fdcc::/16",
"-p", "tcp",
"--dport", strconv.Itoa(constants.MachineAPIPort),
"-j", "ACCEPT",
}
// Allow Corrosion gossip traffic from cluster machines via the management IPv6 WireGuard network.
acceptCorrosionGossipRule := []string{
"-i", network.WireGuardInterfaceName,
"-s", "fdcc::/16",
"-p", "udp",
"--dport", strconv.Itoa(corroservice.DefaultGossipPort),
"-j", "ACCEPT",
}
for _, rule := range [][]string{acceptMachineAPIRule, acceptCorrosionGossipRule} {
if err = ipt6.ProgramRule(iptables.Filter, UncloudInputChain, iptables.Insert, rule); err != nil {
return fmt.Errorf("insert ip6tables rule '%s': %w", strings.Join(rule, " "), err)
}
}
return nil
}
// createIptablesChains ensures UNCLOUD-INPUT iptables and ip6tables chains exist and
// there are jump rules from the main INPUT chains.
func createIptablesChains() error {
ipt4 := iptables.GetIptable(iptables.IPv4)
ipt6 := iptables.GetIptable(iptables.IPv6)
for i, ipt := range []*iptables.IPTable{ipt4, ipt6} {
iptBin := "iptables"
if i == 1 {
iptBin = "ip6tables"
}
// Ensure UNCLOUD-INPUT chain exists. All existing rules are flushed.
if _, err := ipt.NewChain(UncloudInputChain, iptables.Filter); err != nil { if _, err := ipt.NewChain(UncloudInputChain, iptables.Filter); err != nil {
return fmt.Errorf("create %s chain '%s': %w", iptBin, UncloudInputChain, err) return fmt.Errorf("create iptables chain '%s': %w", UncloudInputChain, err)
} }
if err := ipt.RawCombinedOutput("-t", string(iptables.Filter), "-F", UncloudInputChain); err != nil { if err := ipt.RawCombinedOutput("-t", string(iptables.Filter), "-F", UncloudInputChain); err != nil {
return fmt.Errorf("flush %s chain '%s': %w", iptBin, UncloudInputChain, err) return fmt.Errorf("flush iptables chain '%s': %w", UncloudInputChain, err)
}
if err := ipt.AddReturnRule(UncloudInputChain); err != nil {
return fmt.Errorf("add the RETURN rule for iptables chain '%s': %w", UncloudInputChain, err)
} }
// Ensure the main INPUT chain has a jump rule to the UNCLOUD-INPUT chain before any DROP/REJECT rules. // Ensure the main iptables INPUT chain has a jump rule to the UNCLOUD-INPUT chain before any DROP/REJECT rules.
jumpRule := []string{"-m", "comment", "--comment", "Uncloud-managed", "-j", UncloudInputChain} jumpRule := []string{"-m", "comment", "--comment", "Uncloud-managed", "-j", UncloudInputChain}
if !ipt.Exists(iptables.Filter, "INPUT", jumpRule...) { if !ipt.Exists(iptables.Filter, "INPUT", jumpRule...) {
// Look for the first DROP/REJECT rule in the INPUT chain. // Look for the first DROP/REJECT rule in the INPUT chain.
out, err := ipt.Raw("-t", string(iptables.Filter), "-L", "INPUT", "--line-numbers") out, err := ipt.Raw("-t", string(iptables.Filter), "-L", "INPUT", "--line-numbers")
if err != nil { if err != nil {
return fmt.Errorf("get %s rules for chain '%s': %w", iptBin, UncloudInputChain, err) return fmt.Errorf("get iptables rules for chain '%s': %w", UncloudInputChain, err)
} }
firstRejectRuleNum := 0 firstRejectRuleNum := 0
@@ -103,60 +53,21 @@ func createIptablesChains() error {
var addJumpRule []string var addJumpRule []string
if firstRejectRuleNum > 0 { if firstRejectRuleNum > 0 {
addJumpRule = append([]string{ addJumpRule = append([]string{"-t", string(iptables.Filter), "-I", "INPUT", strconv.Itoa(firstRejectRuleNum)},
"-t", string(iptables.Filter), jumpRule...)
"-I", "INPUT",
strconv.Itoa(firstRejectRuleNum),
}, jumpRule...)
} else { } else {
addJumpRule = append([]string{ addJumpRule = append([]string{"-t", string(iptables.Filter), "-A", "INPUT"}, jumpRule...)
"-t", string(iptables.Filter),
"-A", "INPUT",
}, jumpRule...)
} }
if err = ipt.RawCombinedOutput(addJumpRule...); err != nil { if err = ipt.RawCombinedOutput(addJumpRule...); err != nil {
return fmt.Errorf("add %s rule '%s': %w", iptBin, strings.Join(addJumpRule, " "), err) return fmt.Errorf("add iptables rule '%s': %w", strings.Join(addJumpRule, " "), err)
} }
} }
}
return nil // Allow WireGuard traffic to the machine.
} acceptWireGuardRule := []string{"-p", "udp", "--dport", strconv.Itoa(network.WireGuardPort), "-j", "ACCEPT"}
err := ipt.ProgramRule(iptables.Filter, UncloudInputChain, iptables.Insert, acceptWireGuardRule)
// CleanupIptablesChains removes the custom iptables chains and rules created by ConfigureIptablesChains. if err != nil {
func CleanupIptablesChains() error { return fmt.Errorf("insert iptables rule '%s': %w", strings.Join(acceptWireGuardRule, " "), err)
ipt4 := iptables.GetIptable(iptables.IPv4)
ipt6 := iptables.GetIptable(iptables.IPv6)
for i, ipt := range []*iptables.IPTable{ipt4, ipt6} {
iptBin := "iptables"
if i == 1 {
iptBin = "ip6tables"
}
// First, remove the jump rule from INPUT chain to UNCLOUD-INPUT.
jumpRule := []string{"-m", "comment", "--comment", "Uncloud-managed", "-j", UncloudInputChain}
if err := ipt.ProgramRule(iptables.Filter, "INPUT", iptables.Delete, jumpRule); err != nil {
return fmt.Errorf("delete %s jump rule from INPUT: %w", iptBin, err)
}
// Flush all rules from UNCLOUD-INPUT chain as it must be empty before deletion.
if err := ipt.RawCombinedOutput("-t", string(iptables.Filter), "-F", UncloudInputChain); err != nil {
// Chain might not exist which is fine.
if !strings.Contains(err.Error(), "No chain") {
return fmt.Errorf("flush %s chain '%s': %w", iptBin, UncloudInputChain, err)
}
}
// Delete the UNCLOUD-INPUT chain.
if err := ipt.RawCombinedOutput("-t", string(iptables.Filter), "-X", UncloudInputChain); err != nil {
// Chain might not exist which is fine.
if !strings.Contains(err.Error(), "No chain") {
return fmt.Errorf("delete %s chain '%s': %w", iptBin, UncloudInputChain, err)
}
} else {
slog.Info(fmt.Sprintf("Deleted %s chain.", iptBin), "chain", UncloudInputChain)
}
} }
return nil return nil
+89 -154
View File
@@ -13,7 +13,6 @@ import (
"path/filepath" "path/filepath"
"slices" "slices"
"strconv" "strconv"
"sync"
"github.com/docker/docker/client" "github.com/docker/docker/client"
"github.com/docker/go-connections/sockets" "github.com/docker/go-connections/sockets"
@@ -24,12 +23,12 @@ import (
apiproxy "github.com/psviderski/uncloud/internal/machine/api/proxy" apiproxy "github.com/psviderski/uncloud/internal/machine/api/proxy"
"github.com/psviderski/uncloud/internal/machine/caddyconfig" "github.com/psviderski/uncloud/internal/machine/caddyconfig"
"github.com/psviderski/uncloud/internal/machine/cluster" "github.com/psviderski/uncloud/internal/machine/cluster"
"github.com/psviderski/uncloud/internal/machine/constants"
"github.com/psviderski/uncloud/internal/machine/corroservice" "github.com/psviderski/uncloud/internal/machine/corroservice"
"github.com/psviderski/uncloud/internal/machine/dns" "github.com/psviderski/uncloud/internal/machine/dns"
machinedocker "github.com/psviderski/uncloud/internal/machine/docker" machinedocker "github.com/psviderski/uncloud/internal/machine/docker"
"github.com/psviderski/uncloud/internal/machine/network" "github.com/psviderski/uncloud/internal/machine/network"
"github.com/psviderski/uncloud/internal/machine/store" "github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/pkg/api"
"github.com/siderolabs/grpc-proxy/proxy" "github.com/siderolabs/grpc-proxy/proxy"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
"google.golang.org/grpc" "google.golang.org/grpc"
@@ -61,9 +60,9 @@ type Config struct {
// DockerClient manages system and user containers using the local Docker daemon. // DockerClient manages system and user containers using the local Docker daemon.
DockerClient *client.Client DockerClient *client.Client
// CaddyConfigDir specifies the directory where the machine generates the Caddy reverse proxy configuration file // CaddyConfigPath specifies where the machine generates the Caddy reverse proxy configuration file for routing
// for routing external traffic to service containers across the internal network. Default is DataDir/caddy. // external traffic to service containers across the internal network. Default is DataDir/caddy/caddy.json.
CaddyConfigDir string CaddyConfigPath string
// DNSUpstreams specifies the upstream DNS servers for the embedded internal DNS server. // DNSUpstreams specifies the upstream DNS servers for the embedded internal DNS server.
DNSUpstreams []netip.AddrPort DNSUpstreams []netip.AddrPort
} }
@@ -128,8 +127,8 @@ func (c *Config) SetDefaults() (*Config, error) {
} }
} }
if cfg.CaddyConfigDir == "" { if cfg.CaddyConfigPath == "" {
cfg.CaddyConfigDir = filepath.Join(cfg.DataDir, "caddy") cfg.CaddyConfigPath = filepath.Join(cfg.DataDir, "caddy", "caddy.json")
} }
return &cfg, nil return &cfg, nil
@@ -150,20 +149,11 @@ type Machine struct {
started chan struct{} started chan struct{}
// initialised is signalled when the machine is configured as a member of a cluster. // initialised is signalled when the machine is configured as a member of a cluster.
initialised chan struct{} initialised chan struct{}
// networkReady is signalled when the Docker network is configured and ready for containers.
networkReady chan struct{}
// resetting is true when the machine is being reset.
resetting bool
// stop cancels the Run method context to stop the machine gracefully.
stop func()
clusterCtrl *clusterController
// store is the cluster store backed by a distributed Corrosion database. // store is the cluster store backed by a distributed Corrosion database.
store *store.Store store *store.Store
cluster *cluster.Cluster cluster *cluster.Cluster
// dockerService provides high-level operations for managing Docker containers. docker *machinedocker.Server
dockerService *machinedocker.Service
dockerServer *machinedocker.Server
// localMachineServer is the gRPC server for the machine API listening on the local Unix socket. // localMachineServer is the gRPC server for the machine API listening on the local Unix socket.
localMachineServer *grpc.Server localMachineServer *grpc.Server
@@ -173,9 +163,6 @@ type Machine struct {
// It proxies requests to the local or remote machine API servers depending on the request targets // It proxies requests to the local or remote machine API servers depending on the request targets
// and aggregates responses. // and aggregates responses.
localProxyServer *grpc.Server localProxyServer *grpc.Server
// mu protects the Machine from concurrent reads and writes.
mu sync.RWMutex
} }
func NewMachine(config *Config) (*Machine, error) { func NewMachine(config *Config) (*Machine, error) {
@@ -223,16 +210,19 @@ func NewMachine(config *Config) (*Machine, error) {
c := cluster.NewCluster(corroStore, corroAdmin) c := cluster.NewCluster(corroStore, corroAdmin)
// Init dependencies for a gRPC Docker server that proxies requests to the local Docker daemon. // Init dependencies for a gRPC Docker server that proxies requests to the local Docker daemon.
dockerCli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return nil, fmt.Errorf("create Docker client: %w", err)
}
dbFilePath := filepath.Join(config.DataDir, DBFileName) dbFilePath := filepath.Join(config.DataDir, DBFileName)
db, err := NewDB(dbFilePath) db, err := NewDB(dbFilePath)
if err != nil { if err != nil {
return nil, fmt.Errorf("init machine database: %w", err) return nil, fmt.Errorf("init machine database: %w", err)
} }
dockerService := machinedocker.NewService(config.DockerClient, db)
// Init a local gRPC proxy server that proxies requests to the local or remote machine API servers. // Init a local gRPC proxy server that proxies requests to the local or remote machine API servers.
proxyDirector := apiproxy.NewDirector(config.MachineSockPath, constants.MachineAPIPort) proxyDirector := apiproxy.NewDirector(config.MachineSockPath, APIPort)
localProxyServer := grpc.NewServer( localProxyServer := grpc.NewServer(
grpc.ForceServerCodecV2(proxy.Codec()), grpc.ForceServerCodecV2(proxy.Codec()),
grpc.UnknownServiceHandler( grpc.UnknownServiceHandler(
@@ -245,10 +235,8 @@ func NewMachine(config *Config) (*Machine, error) {
state: state, state: state,
started: make(chan struct{}), started: make(chan struct{}),
initialised: make(chan struct{}, 1), initialised: make(chan struct{}, 1),
networkReady: make(chan struct{}),
store: corroStore, store: corroStore,
cluster: c, cluster: c,
dockerService: dockerService,
localProxyServer: localProxyServer, localProxyServer: localProxyServer,
proxyDirector: proxyDirector, proxyDirector: proxyDirector,
} }
@@ -257,10 +245,8 @@ func NewMachine(config *Config) (*Machine, error) {
internalDNSIP := func() netip.Addr { internalDNSIP := func() netip.Addr {
return m.IP() return m.IP()
} }
m.dockerServer = machinedocker.NewServer(dockerService, db, internalDNSIP, m.docker = machinedocker.NewServer(dockerCli, db, internalDNSIP)
machinedocker.WithNetworkReady(m.IsNetworkReady), m.localMachineServer = newGRPCServer(m, c, m.docker)
machinedocker.WithWaitForNetworkReady(m.WaitForNetworkReady))
m.localMachineServer = newGRPCServer(m, c, m.dockerServer)
if m.Initialised() { if m.Initialised() {
m.initialised <- struct{}{} m.initialised <- struct{}{}
@@ -301,9 +287,6 @@ func (m *Machine) IP() netip.Addr {
} }
func (m *Machine) Run(ctx context.Context) error { func (m *Machine) Run(ctx context.Context) error {
// Create a cancellable context for the Run method to allow stopping the machine gracefully.
ctx, m.stop = context.WithCancel(ctx)
// Docker dependency is essential for the machine to function. Block until it's ready. // Docker dependency is essential for the machine to function. Block until it's ready.
if err := docker.WaitDaemonReady(ctx, m.config.DockerClient); err != nil { if err := docker.WaitDaemonReady(ctx, m.config.DockerClient); err != nil {
return fmt.Errorf("wait for Docker daemon: %w", err) return fmt.Errorf("wait for Docker daemon: %w", err)
@@ -311,7 +294,7 @@ func (m *Machine) Run(ctx context.Context) error {
// Configure and start the corrosion service on the loopback if the machine is not initialised as a cluster // Configure and start the corrosion service on the loopback if the machine is not initialised as a cluster
// member. This provides the store required for the machine to initialise a new cluster on it. Once the machine // member. This provides the store required for the machine to initialise a new cluster on it. Once the machine
// is initialised, the corrosion service is managed by the clusterController. // is initialised, the corrosion service is managed by the networkController.
if !m.Initialised() { if !m.Initialised() {
if err := m.configureCorrosion(); err != nil { if err := m.configureCorrosion(); err != nil {
return fmt.Errorf("configure corrosion service: %w", err) return fmt.Errorf("configure corrosion service: %w", err)
@@ -321,7 +304,6 @@ func (m *Machine) Run(ctx context.Context) error {
if err := m.config.CorrosionService.Start(ctx); err != nil { if err := m.config.CorrosionService.Start(ctx); err != nil {
return fmt.Errorf("start corrosion service: %w", err) return fmt.Errorf("start corrosion service: %w", err)
} }
slog.Info("Corrosion service started.")
} }
// Use an errgroup to coordinate error handling and graceful shutdown of multiple machine components. // Use an errgroup to coordinate error handling and graceful shutdown of multiple machine components.
@@ -332,49 +314,64 @@ func (m *Machine) Run(ctx context.Context) error {
if err != nil { if err != nil {
return fmt.Errorf("listen machine API unix socket %q: %w", m.config.MachineSockPath, err) return fmt.Errorf("listen machine API unix socket %q: %w", m.config.MachineSockPath, err)
} }
errGroup.Go(func() error { errGroup.Go(
func() error {
slog.Info("Starting local machine API server.", "path", m.config.MachineSockPath) slog.Info("Starting local machine API server.", "path", m.config.MachineSockPath)
if err := m.localMachineServer.Serve(machineListener); err != nil { if err := m.localMachineServer.Serve(machineListener); err != nil {
return fmt.Errorf("local machine API server failed: %w", err) return fmt.Errorf("local machine API server failed: %w", err)
} }
return nil return nil
}) },
)
// Start the local API proxy server. // Start the local API proxy server.
proxyListener, err := listenUnixSocket(m.config.UncloudSockPath) proxyListener, err := listenUnixSocket(m.config.UncloudSockPath)
if err != nil { if err != nil {
return fmt.Errorf("listen API proxy unix socket %q: %w", m.config.UncloudSockPath, err) return fmt.Errorf("listen API proxy unix socket %q: %w", m.config.UncloudSockPath, err)
} }
errGroup.Go(func() error { errGroup.Go(
func() error {
slog.Info("Starting local API proxy server.", "path", m.config.UncloudSockPath) slog.Info("Starting local API proxy server.", "path", m.config.UncloudSockPath)
if err := m.localProxyServer.Serve(proxyListener); err != nil { if err := m.localProxyServer.Serve(proxyListener); err != nil {
return fmt.Errorf("local API proxy server failed: %w", err) return fmt.Errorf("local API proxy server failed: %w", err)
} }
return nil return nil
}) },
)
// Signal that the machine is ready. // Signal that the machine is ready.
close(m.started) close(m.started)
// Wait for the machine to be initialised as a member of a cluster and run the cluster controller. // Control loop for managing components that depend on the machine being initialised as a cluster member.
errGroup.Go(func() error { errGroup.Go(
func() error {
if !m.Initialised() { if !m.Initialised() {
slog.Info( slog.Info(
"Waiting for the machine to be initialised as a member of a cluster to start the cluster controller.", "Waiting for the machine to be initialised as a member of a cluster " +
"to start the network controller.",
) )
} }
var ctrl *networkController
// Error channel for communicating the termination of the network controller.
errCh := make(chan error)
for {
select { select {
// Wait for the machine to be initialised as a member of a cluster to start the network controller.
// It can be reset when leaving the cluster and then re-initialised again with a new configuration.
case <-m.initialised: case <-m.initialised:
var err error
m.cluster.UpdateMachineID(m.state.ID) m.cluster.UpdateMachineID(m.state.ID)
// Ensure the corrosion config is up to date, including a new gossip address if the machine // Ensure the corrosion config is up to date, including a new gossip address if the machine
// has just joined a cluster. // has just joined a cluster.
if err := m.configureCorrosion(); err != nil { if err = m.configureCorrosion(); err != nil {
return fmt.Errorf("configure corrosion service: %w", err) return fmt.Errorf("configure corrosion service: %w", err)
} }
slog.Info("Configured corrosion service.", "dir", m.config.CorrosionDir) slog.Info("Configured corrosion service.", "dir", m.config.CorrosionDir)
slog.Info("Starting cluster controller.") slog.Info("Starting network controller.")
// Update the proxy director's local address to the machine's management IP address, allowing // Update the proxy director's local address to the machine's management IP address, allowing
// the proxy to identify which requests should be proxied to the local machine API server. // the proxy to identify which requests should be proxied to the local machine API server.
m.proxyDirector.UpdateLocalAddress(m.state.Network.ManagementIP.String()) m.proxyDirector.UpdateLocalAddress(m.state.Network.ManagementIP.String())
@@ -385,11 +382,11 @@ func (m *Machine) Run(ctx context.Context) error {
), ),
) )
// Create a new caddyconfig controller for managing the Caddy reverse proxy configuration. // Create a new Caddyfile controller for managing the Caddy reverse proxy configuration.
// It will also serve the current machine ID at /.uncloud-verify to verify Caddy reachability. // It will also serve the current machine ID at /.uncloud-verify to verify Caddy reachability.
caddyconfigCtrl, err := caddyconfig.NewController(m.store, m.config.CaddyConfigDir, m.state.ID) caddyfileCtrl, err := caddyconfig.NewController(m.store, m.config.CaddyConfigPath, m.state.ID)
if err != nil { if err != nil {
return fmt.Errorf("create caddyconfig controller: %w", err) return fmt.Errorf("create Caddyfile controller: %w", err)
} }
dnsResolver := dns.NewClusterResolver(m.store) dnsResolver := dns.NewClusterResolver(m.store)
@@ -398,39 +395,49 @@ func (m *Machine) Run(ctx context.Context) error {
return fmt.Errorf("create embedded DNS server: %w", err) return fmt.Errorf("create embedded DNS server: %w", err)
} }
m.mu.Lock() ctrl, err = newNetworkController(
m.clusterCtrl, err = newClusterController(
m.state, m.state,
m.store, m.store,
proxyServer, proxyServer,
m.config.CorrosionService, m.config.CorrosionService,
m.dockerService, m.config.DockerClient,
m.networkReady, caddyfileCtrl,
caddyconfigCtrl,
dnsServer, dnsServer,
dnsResolver, dnsResolver,
) )
m.mu.Unlock()
if err != nil { if err != nil {
return fmt.Errorf("initialise cluster controller: %w", err) return fmt.Errorf("initialise network controller: %w", err)
} }
if err = m.clusterCtrl.Run(ctx); err != nil { go func() {
return fmt.Errorf("run cluster controller: %w", err) if err = ctrl.Run(ctx); err != nil {
errCh <- fmt.Errorf("run network controller: %w", err)
} else {
slog.Info("Network controller stopped.")
errCh <- nil
} }
slog.Info("Cluster controller stopped.") }()
case err := <-errCh:
if err != nil {
return err
}
ctrl = nil
case <-ctx.Done(): case <-ctx.Done():
// The context was cancelled before the machine was initialised. // Wait for the network controller to stop before returning.
if ctrl != nil {
if err := <-errCh; err != nil {
return err
}
} }
return nil return nil
}) }
}
},
)
// Shutdown goroutine. // Shutdown goroutine.
errGroup.Go(func() error { errGroup.Go(
var err error func() error {
<-ctx.Done() <-ctx.Done()
slog.Info("Stopping local machine API server.") slog.Info("Stopping local machine API server.")
// TODO: implement timeout for graceful shutdown. // TODO: implement timeout for graceful shutdown.
@@ -444,17 +451,10 @@ func (m *Machine) Run(ctx context.Context) error {
m.proxyDirector.Close() m.proxyDirector.Close()
slog.Info("Local API proxy server stopped.") slog.Info("Local API proxy server stopped.")
// Clean up the machine data and resources if the machine shutdown was initiated by a reset.
if m.resetting {
slog.Info("Cleaning up machine data and resources.")
if err = m.cleanup(); err != nil {
slog.Error("Failed to clean up machine data and resources.", "err", err)
}
}
m.config.DockerClient.Close() m.config.DockerClient.Close()
return err return nil
}) },
)
return errGroup.Wait() return errGroup.Wait()
} }
@@ -483,7 +483,7 @@ func listenUnixSocket(path string) (net.Listener, error) {
// Ensure the parent directory exists and has the correct group permissions. // Ensure the parent directory exists and has the correct group permissions.
parent, _ := filepath.Split(path) parent, _ := filepath.Split(path)
if err = os.MkdirAll(parent, 0o750); err != nil { if err = os.MkdirAll(parent, 0750); err != nil {
return nil, fmt.Errorf("create directory %q: %w", parent, err) return nil, fmt.Errorf("create directory %q: %w", parent, err)
} }
if err = os.Chown(parent, -1, gid); err != nil { if err = os.Chown(parent, -1, gid); err != nil {
@@ -537,38 +537,15 @@ func (m *Machine) configureCorrosion() error {
return fmt.Errorf("write corrosion config: %w", err) return fmt.Errorf("write corrosion config: %w", err)
} }
if err := os.WriteFile(schemaPath, []byte(store.Schema), 0o644); err != nil { if err := os.WriteFile(schemaPath, []byte(store.Schema), 0644); err != nil {
return fmt.Errorf("write corrosion schema: %w", err) return fmt.Errorf("write corrosion schema: %w", err)
} }
return nil return nil
} }
// cleanup removes the machine resources and persistent state.
func (m *Machine) cleanup() error {
var errs []error
m.mu.RLock()
clusterCtrl := m.clusterCtrl
m.mu.RUnlock()
if clusterCtrl != nil {
if err := clusterCtrl.Cleanup(); err != nil {
errs = append(errs, fmt.Errorf("cleanup cluster resources: %w", err))
}
}
if err := os.RemoveAll(m.config.DataDir); err != nil {
errs = append(errs,
fmt.Errorf("remove data directory with persistent machine state '%s': %w", m.config.DataDir, err))
} else {
slog.Info("Removed data directory storing persistent machine state.", "path", m.config.DataDir)
}
return errors.Join(errs...)
}
// CheckPrerequisites verifies if the machine meets all necessary system requirements to participate in the cluster. // CheckPrerequisites verifies if the machine meets all necessary system requirements to participate in the cluster.
func (m *Machine) CheckPrerequisites(_ context.Context, _ *emptypb.Empty) (*pb.CheckPrerequisitesResponse, error) { func (m *Machine) CheckPrerequisites(ctx context.Context, _ *emptypb.Empty) (*pb.CheckPrerequisitesResponse, error) {
// Check DNS port (UDP) availability. // Check DNS port (UDP) availability.
if err := checkDNSPortAvailable(); err != nil { if err := checkDNSPortAvailable(); err != nil {
return &pb.CheckPrerequisitesResponse{ return &pb.CheckPrerequisitesResponse{
@@ -799,62 +776,20 @@ func (m *Machine) Inspect(_ context.Context, _ *emptypb.Empty) (*pb.MachineInfo,
}, nil }, nil
} }
// IsNetworkReady returns true if the Docker network is ready for containers. // Reset restores the machine to a clean state, removing all cluster-related сonfiguration and data and scheduling
func (m *Machine) IsNetworkReady() bool { // a graceful shutdown. The uncloud daemon will restart the machine if managed by systemd.
if !m.Initialised() { func (m *Machine) Reset(ctx context.Context, _ *pb.ResetRequest) (*emptypb.Empty, error) {
// If machine is not initialized, there's no network to check
return false
}
// Check if network is ready by checking if the networkReady channel has been closed
select {
case <-m.networkReady:
return true
default:
return false
}
}
// WaitForNetworkReady waits for the Docker network to be ready for containers.
// It returns nil when the network is ready or an error if the context is cancelled.
func (m *Machine) WaitForNetworkReady(ctx context.Context) error {
if !m.Initialised() {
// If machine is not initialized, there's no network to wait for
return nil
}
// Wait for network to be ready or context to be cancelled
select {
case <-m.networkReady:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// Reset restores the machine to a clean state, scheduling a graceful shutdown and removing all cluster-related
// configuration and resource. The uncloud daemon will restart the machine if managed by systemd.
func (m *Machine) Reset(_ context.Context, _ *pb.ResetRequest) (*emptypb.Empty, error) {
if !m.Initialised() {
return nil, nil
}
// Check if the machine is already being reset to avoid concurrent resets.
m.mu.Lock()
if m.resetting {
m.mu.Unlock()
return nil, status.Error(codes.FailedPrecondition, "machine is already being reset")
}
m.resetting = true
m.mu.Unlock()
slog.Info("Resetting machine to a clean state.") slog.Info("Resetting machine to a clean state.")
// Trigger the machine shutdown. The resetting boolean informs the machine to clean up its resources on shutdown.
// We can't clean up the resources synchronously here because this is an RPC call that depends on the running
// gRPC server and network.
m.stop()
return &emptypb.Empty{}, nil // TODO: stop and remove all managed service containers.
// TODO: check if the request is coming from the unix or network socket. For the network socket, the reset should
// be called in a separate goroutine to avoid blocking the RPC response.
// TODO: stop the network controller
// TODO: implement and call Cleanup on the network controller to remove Docker network, WG interface, iptables
// rules, corrosion state, ?stop corrosion service.
// TODO: stop the machine and remove the machine.json state. The daemon should restart it to a clean state.
return &emptypb.Empty{}, status.Error(codes.Unimplemented, "reset machine is not implemented yet")
} }
// InspectService returns detailed information about a service and its containers stored in the cluster store. // InspectService returns detailed information about a service and its containers stored in the cluster store.
@@ -889,7 +824,7 @@ func (m *Machine) InspectService(
} }
} }
ctr := records[0].Container ctr := api.ServiceContainer{Container: records[0].Container}
svc := &pb.Service{ svc := &pb.Service{
Id: ctr.ServiceID(), Id: ctr.ServiceID(),
Name: ctr.ServiceName(), Name: ctr.ServiceName(),
@@ -12,9 +12,9 @@ import (
"time" "time"
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
"github.com/docker/docker/client"
"github.com/psviderski/uncloud/internal/machine/api/pb" "github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/caddyconfig" "github.com/psviderski/uncloud/internal/machine/caddyconfig"
"github.com/psviderski/uncloud/internal/machine/constants"
"github.com/psviderski/uncloud/internal/machine/corroservice" "github.com/psviderski/uncloud/internal/machine/corroservice"
"github.com/psviderski/uncloud/internal/machine/dns" "github.com/psviderski/uncloud/internal/machine/dns"
"github.com/psviderski/uncloud/internal/machine/docker" "github.com/psviderski/uncloud/internal/machine/docker"
@@ -25,10 +25,11 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
) )
// clusterController is the main controller for the machine that is a cluster member. It manages components such as const (
// the WireGuard network, API server listening the WireGuard network, Corrosion service, Docker network and containers, APIPort = 51000
// and others. )
type clusterController struct {
type networkController struct {
state *State state *State
store *store.Store store *store.Store
@@ -37,30 +38,26 @@ type clusterController struct {
server *grpc.Server server *grpc.Server
corroService corroservice.Service corroService corroservice.Service
dockerCtrl *docker.Controller dockerCli *client.Client
// dockerReady is signalled when Docker is configured and ready for containers. caddyfileCtrl *caddyconfig.Controller
dockerReady chan<- struct{}
caddyconfigCtrl *caddyconfig.Controller
// dnsServer is the embedded internal DNS server for the cluster listening on the machine IP. // dnsServer is the embedded internal DNS server for the cluster listening on the machine IP.
dnsServer *dns.Server dnsServer *dns.Server
dnsResolver *dns.ClusterResolver dnsResolver *dns.ClusterResolver
// stopped is a channel that is closed when the controller is stopped.
stopped chan struct{}
} }
func newClusterController( func newNetworkController(
state *State, state *State,
store *store.Store, store *store.Store,
server *grpc.Server, server *grpc.Server,
corroService corroservice.Service, corroService corroservice.Service,
dockerService *docker.Service, dockerCli *client.Client,
dockerReady chan<- struct{},
caddyfileCtrl *caddyconfig.Controller, caddyfileCtrl *caddyconfig.Controller,
dnsServer *dns.Server, dnsServer *dns.Server,
dnsResolver *dns.ClusterResolver, dnsResolver *dns.ClusterResolver,
) (*clusterController, error) { ) (
*networkController, error,
) {
slog.Info("Starting WireGuard network.") slog.Info("Starting WireGuard network.")
wgnet, err := network.NewWireGuardNetwork() wgnet, err := network.NewWireGuardNetwork()
if err != nil { if err != nil {
@@ -68,97 +65,88 @@ func newClusterController(
} }
endpointChanges := wgnet.WatchEndpoints() endpointChanges := wgnet.WatchEndpoints()
return &clusterController{ return &networkController{
state: state, state: state,
store: store, store: store,
wgnet: wgnet, wgnet: wgnet,
endpointChanges: endpointChanges, endpointChanges: endpointChanges,
server: server, server: server,
corroService: corroService, corroService: corroService,
dockerCtrl: docker.NewController(state.ID, dockerService, store), dockerCli: dockerCli,
dockerReady: dockerReady, caddyfileCtrl: caddyfileCtrl,
caddyconfigCtrl: caddyfileCtrl,
dnsServer: dnsServer, dnsServer: dnsServer,
dnsResolver: dnsResolver, dnsResolver: dnsResolver,
stopped: make(chan struct{}),
}, nil }, nil
} }
func (cc *clusterController) Run(ctx context.Context) error { func (nc *networkController) Run(ctx context.Context) error {
defer close(cc.stopped)
if err := firewall.ConfigureIptablesChains(); err != nil { if err := firewall.ConfigureIptablesChains(); err != nil {
return fmt.Errorf("configure iptables chains: %w", err) return fmt.Errorf("configure iptables chains: %w", err)
} }
if err := cc.ensureDockerNetwork(ctx); err != nil { if err := nc.wgnet.Configure(*nc.state.Network); err != nil {
return err
}
slog.Info("Docker network configured.")
if err := cc.wgnet.Configure(*cc.state.Network); err != nil {
return fmt.Errorf("configure WireGuard network: %w", err) return fmt.Errorf("configure WireGuard network: %w", err)
} }
slog.Info("WireGuard network configured.") slog.Info("WireGuard network configured.")
if cc.corroService.Running() { if nc.corroService.Running() {
// Corrosion service was running before the WireGuard network was configured so we need to restart it. // Corrosion service was running before the WireGuard network was configured so we need to restart it.
slog.Info("Restarting corrosion service to apply new configuration with WireGuard network.") slog.Info("Restarting corrosion service to apply new configuration with WireGuard network.")
if err := cc.corroService.Restart(ctx); err != nil { if err := nc.corroService.Restart(ctx); err != nil {
return fmt.Errorf("restart corrosion service: %w", err) return fmt.Errorf("restart corrosion service: %w", err)
} }
slog.Info("Corrosion service restarted.")
} else { } else {
slog.Info("Starting corrosion service.") slog.Info("Starting corrosion service.")
if err := cc.corroService.Start(ctx); err != nil { if err := nc.corroService.Start(ctx); err != nil {
return fmt.Errorf("start corrosion service: %w", err) return fmt.Errorf("start corrosion service: %w", err)
} }
slog.Info("Corrosion service started.")
} }
// TODO: Figure out if we need to manually stop the corrosion service when the context is done or just
// rely on systemd to handle service dependencies on its own.
errGroup, ctx := errgroup.WithContext(ctx) errGroup, ctx := errgroup.WithContext(ctx)
// Start the network API server. Assume the management IP can't be changed when the network is running. // Start the network API server. Assume the management IP can't be changed when the network is running.
apiAddr := net.JoinHostPort(cc.state.Network.ManagementIP.String(), strconv.Itoa(constants.MachineAPIPort)) apiAddr := net.JoinHostPort(nc.state.Network.ManagementIP.String(), strconv.Itoa(APIPort))
listener, err := net.Listen("tcp", apiAddr) listener, err := net.Listen("tcp", apiAddr)
if err != nil { if err != nil {
return fmt.Errorf("listen API port: %w", err) return fmt.Errorf("listen API port: %w", err)
} }
errGroup.Go(func() error { errGroup.Go(
func() error {
slog.Info("Starting network API server.", "addr", apiAddr) slog.Info("Starting network API server.", "addr", apiAddr)
if err := cc.server.Serve(listener); err != nil { if err := nc.server.Serve(listener); err != nil {
return fmt.Errorf("network API server failed: %w", err) return fmt.Errorf("network API server failed: %w", err)
} }
return nil return nil
}) },
)
errGroup.Go(func() error { errGroup.Go(func() error {
slog.Info("Starting embedded DNS resolver.") slog.Info("Starting embedded DNS resolver.")
if err := cc.dnsResolver.Run(ctx); err != nil { if err := nc.dnsResolver.Run(ctx); err != nil {
return fmt.Errorf("embedded DNS resolver failed: %w", err) return fmt.Errorf("embedded DNS resolver failed: %w", err)
} }
return nil return nil
}) })
// The Docker network must be created before starting the DNS server because it listens on the machine IP.
errGroup.Go(func() error { errGroup.Go(func() error {
slog.Info("Starting embedded DNS server.") slog.Info("Starting embedded DNS server.")
if err := cc.dnsServer.Run(ctx); err != nil { if err := nc.dnsServer.Run(ctx); err != nil {
return fmt.Errorf("embedded DNS server failed: %w", err) return fmt.Errorf("embedded DNS server failed: %w", err)
} }
return nil return nil
}) })
// Synchronise Docker containers to the cluster store. // Setup Docker network and synchronise containers to the cluster store.
errGroup.Go(func() error { errGroup.Go(func() error {
slog.Info("Watching Docker containers and syncing them to cluster store.") return nc.prepareAndWatchDocker(ctx)
return cc.syncDockerContainers(ctx)
}) })
// Handle machine changes in the cluster. Handling machine and endpoint changes should be done // Handle machine changes in the cluster. Handling machine and endpoint changes should be done
// in separate goroutines to avoid a deadlock when reconfiguring the network. // in separate goroutines to avoid a deadlock when reconfiguring the network.
errGroup.Go(func() error { errGroup.Go(func() error {
if err := cc.handleMachineChanges(ctx); err != nil { if err := nc.handleMachineChanges(ctx); err != nil {
return fmt.Errorf("handle new machines: %w", err) return fmt.Errorf("handle new machines: %w", err)
} }
return nil return nil
@@ -168,24 +156,24 @@ func (cc *clusterController) Run(ctx context.Context) error {
errGroup.Go(func() error { errGroup.Go(func() error {
for { for {
select { select {
case e, ok := <-cc.endpointChanges: case e, ok := <-nc.endpointChanges:
if !ok { if !ok {
// The channel was closed, stop watching for changes. // The channel was closed, stop watching for changes.
cc.endpointChanges = nil nc.endpointChanges = nil
return nil return nil
} }
cc.state.mu.Lock() nc.state.mu.Lock()
for i := range cc.state.Network.Peers { for i := range nc.state.Network.Peers {
if cc.state.Network.Peers[i].PublicKey.Equal(e.PublicKey) { if nc.state.Network.Peers[i].PublicKey.Equal(e.PublicKey) {
cc.state.Network.Peers[i].Endpoint = &e.Endpoint nc.state.Network.Peers[i].Endpoint = &e.Endpoint
break break
} }
} }
if err := cc.state.Save(); err != nil { if err := nc.state.Save(); err != nil {
slog.Error("Failed to save machine state.", "err", err) slog.Error("Failed to save machine state.", "err", err)
} }
cc.state.mu.Unlock() nc.state.mu.Unlock()
slog.Debug("Preserved endpoint change in the machine state.", slog.Debug("Preserved endpoint change in the machine state.",
"public_key", e.PublicKey, "endpoint", e.Endpoint) "public_key", e.PublicKey, "endpoint", e.Endpoint)
@@ -196,66 +184,48 @@ func (cc *clusterController) Run(ctx context.Context) error {
}) })
errGroup.Go(func() error { errGroup.Go(func() error {
if err := cc.wgnet.Run(ctx); err != nil { if err := nc.wgnet.Run(ctx); err != nil {
return fmt.Errorf("WireGuard network failed: %w", err) return fmt.Errorf("WireGuard network failed: %w", err)
} }
return nil return nil
}) })
errGroup.Go(func() error { errGroup.Go(func() error {
slog.Info("Starting caddyconfig controller.") slog.Info("Starting Caddyconfig controller.")
if err := cc.caddyconfigCtrl.Run(ctx); err != nil { if err := nc.caddyfileCtrl.Run(ctx); err != nil {
return fmt.Errorf("caddyconfig controller failed: %w", err) //goland:noinspection GoErrorStringFormat
return fmt.Errorf("Caddyconfig controller failed: %w", err)
} }
return nil return nil
}) })
// Wait for the context to be done and stop the network API server. // Wait for the context to be done and stop the network API server.
errGroup.Go(func() error {
<-ctx.Done() <-ctx.Done()
slog.Info("Stopping network API server.") slog.Info("Stopping network API server.")
// TODO: implement timeout for graceful shutdown. // TODO: implement timeout for graceful shutdown.
cc.server.GracefulStop() nc.server.GracefulStop()
slog.Info("Network API server stopped.") slog.Info("Network API server stopped.")
return nil
})
// Wait for all controllers to finish. return errGroup.Wait()
err = errGroup.Wait()
// It's safe to stop the Corrosion service after the controllers depending on it and API server are stopped.
// Use a new context with a timeout as the current context is already canceled.
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if corroErr := cc.corroService.Stop(ctx); corroErr != nil {
err = errors.Join(err, fmt.Errorf("stop corrosion service: %w", corroErr))
} else {
slog.Info("Corrosion service stopped.")
} }
return err // prepareAndWatchDocker configures the Docker network and watches local Docker containers to sync them
} // to the cluster store.
func (nc *networkController) prepareAndWatchDocker(ctx context.Context) error {
// ensureDockerNetwork ensures that the Docker network is configured and ready for containers. manager := docker.NewManager(nc.dockerCli, nc.state.ID, nc.store)
func (cc *clusterController) ensureDockerNetwork(ctx context.Context) error { if err := manager.WaitDaemonReady(ctx); err != nil {
if err := cc.dockerCtrl.WaitDaemonReady(ctx); err != nil {
return fmt.Errorf("wait for Docker daemon: %w", err) return fmt.Errorf("wait for Docker daemon: %w", err)
} }
if err := cc.dockerCtrl.EnsureUncloudNetwork( if err := manager.EnsureUncloudNetwork(ctx, nc.state.Network.Subnet, nc.dnsServer.ListenAddr()); err != nil {
ctx,
cc.state.Network.Subnet,
cc.dnsServer.ListenAddr(),
); err != nil {
return fmt.Errorf("ensure Docker network: %w", err) return fmt.Errorf("ensure Docker network: %w", err)
} }
slog.Info("Docker network configured.")
// Signal that Docker is ready for containers. slog.Info("Watching Docker containers and syncing them to cluster store.")
close(cc.dockerReady)
return nil
}
// syncDockerContainers watches local Docker containers and syncs them to the cluster store.
// TODO: move this to the Docker controller.
func (cc *clusterController) syncDockerContainers(ctx context.Context) error {
// Retry to watch and sync containers until the context is done. // Retry to watch and sync containers until the context is done.
boff := backoff.WithContext(backoff.NewExponentialBackOff( boff := backoff.WithContext(backoff.NewExponentialBackOff(
backoff.WithInitialInterval(100*time.Millisecond), backoff.WithInitialInterval(100*time.Millisecond),
@@ -263,7 +233,7 @@ func (cc *clusterController) syncDockerContainers(ctx context.Context) error {
backoff.WithMaxElapsedTime(0), backoff.WithMaxElapsedTime(0),
), ctx) ), ctx)
watchAndSync := func() error { watchAndSync := func() error {
if wErr := cc.dockerCtrl.WatchAndSyncContainers(ctx); wErr != nil { if wErr := manager.WatchAndSyncContainers(ctx); wErr != nil {
slog.Error("Failed to watch and sync containers to cluster store, retrying.", "err", wErr) slog.Error("Failed to watch and sync containers to cluster store, retrying.", "err", wErr)
return wErr return wErr
} }
@@ -281,7 +251,7 @@ func (cc *clusterController) syncDockerContainers(ctx context.Context) error {
// handleMachineChanges subscribes to machine changes in the cluster and reconfigures the network peers accordingly // handleMachineChanges subscribes to machine changes in the cluster and reconfigures the network peers accordingly
// when changes occur. // when changes occur.
func (cc *clusterController) handleMachineChanges(ctx context.Context) error { func (nc *networkController) handleMachineChanges(ctx context.Context) error {
for { for {
// Retry to subscribe to machine changes indefinitely until the context is done. // Retry to subscribe to machine changes indefinitely until the context is done.
boff := backoff.WithContext(backoff.NewExponentialBackOff( boff := backoff.WithContext(backoff.NewExponentialBackOff(
@@ -296,7 +266,7 @@ func (cc *clusterController) handleMachineChanges(ctx context.Context) error {
err error err error
) )
subscribe := func() error { subscribe := func() error {
if machines, changes, err = cc.store.SubscribeMachines(ctx); err != nil { if machines, changes, err = nc.store.SubscribeMachines(ctx); err != nil {
slog.Info("Failed to subscribe to machine changes, retrying.", "err", err) slog.Info("Failed to subscribe to machine changes, retrying.", "err", err)
} }
return err return err
@@ -314,7 +284,7 @@ func (cc *clusterController) handleMachineChanges(ctx context.Context) error {
// completes. Skip configuration now and apply it when the store changes are received. // completes. Skip configuration now and apply it when the store changes are received.
if len(machines) > 0 { if len(machines) > 0 {
slog.Info("Reconfiguring network peers with the current machines.", "machines", len(machines)) slog.Info("Reconfiguring network peers with the current machines.", "machines", len(machines))
if err = cc.configurePeers(machines); err != nil { if err = nc.configurePeers(machines); err != nil {
slog.Error("Failed to configure peers.", "err", err) slog.Error("Failed to configure peers.", "err", err)
} }
} }
@@ -326,11 +296,11 @@ func (cc *clusterController) handleMachineChanges(ctx context.Context) error {
// be reworked as well. // be reworked as well.
case <-changes: case <-changes:
slog.Info("Cluster machines changed, reconfiguring network peers.") slog.Info("Cluster machines changed, reconfiguring network peers.")
if machines, err = cc.store.ListMachines(ctx); err != nil { if machines, err = nc.store.ListMachines(ctx); err != nil {
slog.Error("Failed to list machines.", "err", err) slog.Error("Failed to list machines.", "err", err)
continue continue
} }
if err = cc.configurePeers(machines); err != nil { if err = nc.configurePeers(machines); err != nil {
slog.Error("Failed to configure peers.", "err", err) slog.Error("Failed to configure peers.", "err", err)
} }
case <-ctx.Done(): case <-ctx.Done():
@@ -340,23 +310,23 @@ func (cc *clusterController) handleMachineChanges(ctx context.Context) error {
} }
} }
func (cc *clusterController) configurePeers(machines []*pb.MachineInfo) error { func (nc *networkController) configurePeers(machines []*pb.MachineInfo) error {
if len(machines) == 0 { if len(machines) == 0 {
return fmt.Errorf("no machines to configure peers") return fmt.Errorf("no machines to configure peers")
} }
cc.state.mu.RLock() nc.state.mu.RLock()
currentPeerEndpoints := make(map[string]*netip.AddrPort, len(cc.state.Network.Peers)) currentPeerEndpoints := make(map[string]*netip.AddrPort, len(nc.state.Network.Peers))
for _, p := range cc.state.Network.Peers { for _, p := range nc.state.Network.Peers {
currentPeerEndpoints[p.PublicKey.String()] = p.Endpoint currentPeerEndpoints[p.PublicKey.String()] = p.Endpoint
} }
cc.state.mu.RUnlock() nc.state.mu.RUnlock()
// Construct the list of peers from the machine configurations ensuring that the current endpoint is preserved. // Construct the list of peers from the machine configurations ensuring that the current endpoint is preserved.
peers := make([]network.PeerConfig, 0, len(machines)-1) peers := make([]network.PeerConfig, 0, len(machines)-1)
for _, m := range machines { for _, m := range machines {
// Skip the current machine. // Skip the current machine.
if m.Id == cc.state.ID { if m.Id == nc.state.ID {
continue continue
} }
if err := m.Network.Validate(); err != nil { if err := m.Network.Validate(); err != nil {
@@ -389,37 +359,20 @@ func (cc *clusterController) configurePeers(machines []*pb.MachineInfo) error {
} }
// Preserve the new list of peers in the machine state. // Preserve the new list of peers in the machine state.
cc.state.mu.Lock() nc.state.mu.Lock()
cc.state.Network.Peers = peers nc.state.Network.Peers = peers
err := cc.state.Save() err := nc.state.Save()
cc.state.mu.Unlock() nc.state.mu.Unlock()
if err != nil { if err != nil {
return fmt.Errorf("save machine state: %w", err) return fmt.Errorf("save machine state: %w", err)
} }
cc.state.mu.RLock() nc.state.mu.RLock()
defer cc.state.mu.RUnlock() defer nc.state.mu.RUnlock()
if err = cc.wgnet.Configure(*cc.state.Network); err != nil { if err = nc.wgnet.Configure(*nc.state.Network); err != nil {
return fmt.Errorf("configure network peers: %w", err) return fmt.Errorf("configure network peers: %w", err)
} }
return nil return nil
} }
// Cleanup cleans up the cluster resources such as the WireGuard network, iptables rules, Docker network and containers. // TODO: method to shutdown network when leaving a cluster. Regular context cancellation shouldn't bring it down.
func (cc *clusterController) Cleanup() error {
// Wait for the controller to stop before cleaning up.
<-cc.stopped
var errs []error
if err := cc.dockerCtrl.Cleanup(); err != nil {
errs = append(errs, fmt.Errorf("cleanup Docker resources: %w", err))
}
if err := cc.wgnet.Cleanup(); err != nil {
errs = append(errs, fmt.Errorf("cleanup WireGuard network: %w", err))
}
if err := firewall.CleanupIptablesChains(); err != nil {
errs = append(errs, fmt.Errorf("cleanup iptables chains: %w", err))
}
return errors.Join(errs...)
}
+1 -1
View File
@@ -14,7 +14,7 @@ func MachineIP(subnet netip.Prefix) netip.Addr {
} }
// ManagementIP returns the IPv6 address of a peer derived from the first 14 bytes of its public key. // ManagementIP returns the IPv6 address of a peer derived from the first 14 bytes of its public key.
// This address always starts with fdcc: and is intended for cluster management traffic. // This address is intended for cluster management traffic.
func ManagementIP(publicKey secret.Secret) netip.Addr { func ManagementIP(publicKey secret.Secret) netip.Addr {
bytes := [16]byte{0xfd, 0xcc} bytes := [16]byte{0xfd, 0xcc}
copy(bytes[2:], publicKey[:14]) copy(bytes[2:], publicKey[:14])

Some files were not shown because too many files have changed in this diff Show More