philipp/uncloud
1
0
Fork 0
mirror of https://github.com/psviderski/uncloud.git synced 2026-08-26 19:13:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: concatenate custom Caddy configs for services into final Caddyfile (no upstream interpolation)

Browse Source
This commit is contained in:
Pasha Sviderski
2025-08-19 22:01:48 +10:00
parent 8bf9fc0c9c
commit 03970862ab
6 changed files with 267 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/machine/caddyconfig/caddyfile.go
+99 -14
View File
@@ -2,13 +2,18 @@ package caddyconfig
import (
"bytes"
"cmp"
"context"
"fmt"
"log/slog"
"maps"
"net"
"slices"
"strconv"
"strings"
"text/template"
"github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/pkg/api"
)
@@ -47,14 +52,26 @@ https://{{$hostname}} {
// CaddyfileGenerator generates a Caddyfile configuration for the Caddy reverse proxy.
type CaddyfileGenerator struct {
// MachineID is the unique identifier of the machine where the controller is running.
MachineID string
Validator CaddyfileValidator
// machineID is the unique identifier of the machine where the controller is running.
machineID string
validator CaddyfileValidator
log *slog.Logger
}
// CaddyfileValidator is an interface for validating Caddyfile configurations.
type CaddyfileValidator interface {
Validate(caddyfile string) error
Validate(ctx context.Context, caddyfile string) error
}
func NewCaddyfileGenerator(machineID string, validator CaddyfileValidator, log *slog.Logger) *CaddyfileGenerator {
if log == nil {
log = slog.Default()
}
return &CaddyfileGenerator{
machineID: machineID,
validator: validator,
log: log,
}
}
// Generate creates a Caddyfile configuration based on the provided service containers.
@@ -70,15 +87,87 @@ type CaddyfileValidator interface {
// [service-a x-caddy]
// ...
// [service-z x-caddy]
func (g *CaddyfileGenerator) Generate(containers []api.ServiceContainer) (string, error) {
baseCaddyfile, err := g.generateBaseFromPorts(containers)
func (g *CaddyfileGenerator) Generate(ctx context.Context, records []store.ContainerRecord) (string, error) {
containers := make([]api.ServiceContainer, len(records))
for i, cr := range records {
containers[i] = cr.Container
}
// Sort containers by service name and container ID to generate a stable Caddyfile.
slices.SortFunc(containers, func(a, b api.ServiceContainer) int {
return cmp.Or(
strings.Compare(a.ServiceName(), b.ServiceName()),
strings.Compare(a.ID, b.ID),
)
})
caddyfile, err := g.generateBaseFromPorts(containers)
if err != nil {
return "", fmt.Errorf("generate base Caddyfile from service ports: %w", err)
}
// TODO: Implement support for custom Caddy configs (x-caddy) in service specs.
// Find the 'caddy' service container on this machine. Use the most recent one if multiple exist.
var caddyCtr *api.ServiceContainer
for _, cr := range records {
if cr.MachineID == g.machineID && cr.Container.ServiceName() == CaddyServiceName &&
(caddyCtr == nil || cr.Container.Created > caddyCtr.Created) {
caddyCtr = &cr.Container
}
}
return baseCaddyfile, nil
// If the caddy container is running on this machine and has a custom Caddy config, prepend it to the generated
// Caddyfile and validate it.
if caddyCtr != nil && caddyCtr.ServiceSpec.CaddyConfig() != "" {
// TODO: render the template actions in the Caddy config.
caddyfileCandidate := caddyCtr.ServiceSpec.CaddyConfig() + "\n\n" + caddyfile
if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil {
g.log.Error("Custom Caddy config on the caddy container on this machine is invalid, skipping it.",
"service", caddyCtr.ServiceName(), "container", caddyCtr.ID, "err", err)
} else {
caddyfile = caddyfileCandidate
}
}
// There could be multiple service containers for the same service with different custom Caddy configs, for example,
// if the service has been partially updated. The most recent container for each service defines the current custom
// Caddy config for that service.
latestServiceContainers := make(map[string]api.ServiceContainer, len(containers))
for _, ctr := range containers {
if latest, ok := latestServiceContainers[ctr.ServiceName()]; ok {
if ctr.Created > latest.Created {
latestServiceContainers[ctr.ServiceName()] = ctr
}
} else {
latestServiceContainers[ctr.ServiceName()] = ctr
}
}
sortedServiceNames := slices.Sorted(maps.Keys(latestServiceContainers))
for _, serviceName := range sortedServiceNames {
// Skip the caddy container as we already processed it.
if serviceName == CaddyServiceName {
continue
}
ctr := latestServiceContainers[serviceName]
if ctr.ServiceSpec.CaddyConfig() == "" {
continue
}
// TODO: render the template actions in the Caddy config.
caddyfileCandidate := fmt.Sprintf("%s\n# Service: %s\n%s\n",
caddyfile, ctr.ServiceName(), ctr.ServiceSpec.CaddyConfig())
if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil {
g.log.Error("Custom Caddy config for service is invalid, skipping it.",
"service", ctr.ServiceName(), "err", err)
continue
}
caddyfile = caddyfileCandidate
}
return caddyfile, nil
}
func (g *CaddyfileGenerator) generateBaseFromPorts(containers []api.ServiceContainer) (string, error) {
@@ -97,7 +186,7 @@ func (g *CaddyfileGenerator) generateBaseFromPorts(containers []api.ServiceConta
HTTPSHostUpstreams map[string][]string
}{
VerifyPath: VerifyPath,
VerifyResponse: g.MachineID,
VerifyResponse: g.machineID,
HTTPHostUpstreams: httpHostUpstreams,
HTTPSHostUpstreams: httpsHostUpstreams,
}
@@ -111,16 +200,12 @@ func (g *CaddyfileGenerator) generateBaseFromPorts(containers []api.ServiceConta
}
// httpUpstreamsFromContainers extracts upstreams for HTTP and HTTPS protocols from the published ports of the provided
// service containers.
// service containers. It's expected that all containers are healthy.
func httpUpstreamsFromContainers(containers []api.ServiceContainer) (map[string][]string, map[string][]string) {
// Maps hostnames to lists of upstreams (container IP:port pairs).
httpHostUpstreams := make(map[string][]string)
httpsHostUpstreams := make(map[string][]string)
for _, ctr := range containers {
if !ctr.Healthy() {
continue
}
ip := ctr.UncloudNetworkIP()
if !ip.IsValid() {
// Container is not connected to the uncloud Docker network (could be host network).
internal/machine/caddyconfig/caddyfile_test.go
+56 -66
View File
@@ -1,8 +1,10 @@
package caddyconfig
import (
"context"
"testing"
"github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/pkg/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -24,25 +26,24 @@ func TestCaddyfileGenerator(t *testing.T) {
}
`
generator := &CaddyfileGenerator{
MachineID: "test-machine-id",
}
// TODO: mock validator
generator := NewCaddyfileGenerator("test-machine-id", nil, nil)
tests := []struct {
name string
containers []api.ServiceContainer
containers []store.ContainerRecord
want string
wantErr bool
}{
{
name: "empty containers",
containers: []api.ServiceContainer{},
containers: []store.ContainerRecord{},
want: caddyfileHeader,
},
{
name: "HTTP container",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "app.example.com:8080/http"),
containers: []store.ContainerRecord{
newContainerRecord(newContainer("10.210.0.2", "app.example.com:8080/http"), "mach1"),
},
want: caddyfileHeader + `
http://app.example.com {
@@ -56,9 +57,9 @@ http://app.example.com {
},
{
name: "load balancing multiple containers",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "app.example.com:8080/http"),
newContainer("10.210.0.3", "app.example.com:8080/http"),
containers: []store.ContainerRecord{
newContainerRecord(newContainer("10.210.0.2", "app.example.com:8080/http"), "mach1"),
newContainerRecord(newContainer("10.210.0.3", "app.example.com:8080/http"), "mach1"),
},
want: caddyfileHeader + `
http://app.example.com {
@@ -72,8 +73,8 @@ http://app.example.com {
},
{
name: "HTTPS container",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "secure.example.com:8000/https"),
containers: []store.ContainerRecord{
newContainerRecord(newContainer("10.210.0.2", "secure.example.com:8000/https"), "mach1"),
},
want: caddyfileHeader + `
https://secure.example.com {
@@ -87,20 +88,32 @@ https://secure.example.com {
},
{
name: "mixed HTTP and HTTPS",
containers: []api.ServiceContainer{
newContainer("10.210.0.2",
"app.example.com:8080/http",
"web.example.com:8000/http"),
newContainer("10.210.0.3",
"app.example.com:8080/http",
"secure.example.com:8888/https"),
newContainer("10.210.0.4",
"web.example.com:8000/http",
"secure.example.com:8888/https"),
newContainer("10.210.0.5",
"app.example.com:8080/http",
"web.example.com:8000/http",
"secure.example.com:8888/https"),
containers: []store.ContainerRecord{
newContainerRecord(
newContainer("10.210.0.2",
"app.example.com:8080/http",
"web.example.com:8000/http"),
"mach1",
),
newContainerRecord(
newContainer("10.210.0.3",
"app.example.com:8080/http",
"secure.example.com:8888/https"),
"mach1",
),
newContainerRecord(
newContainer("10.210.0.4",
"web.example.com:8000/http",
"secure.example.com:8888/https"),
"mach1",
),
newContainerRecord(
newContainer("10.210.0.5",
"app.example.com:8080/http",
"web.example.com:8000/http",
"secure.example.com:8888/https"),
"mach1",
),
},
want: caddyfileHeader + `
http://app.example.com {
@@ -130,63 +143,33 @@ https://secure.example.com {
},
{
name: "container without uncloud network ignored",
containers: []api.ServiceContainer{
newContainerWithoutNetwork("ignored.example.com:8080/http"),
containers: []store.ContainerRecord{
newContainerRecord(newContainerWithoutNetwork("ignored.example.com:8080/http"), "mach1"),
},
want: caddyfileHeader,
},
{
name: "container with invalid port ignored",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "invalid-port"),
containers: []store.ContainerRecord{
newContainerRecord(newContainer("10.210.0.2", "invalid-port"), "mach1"),
},
want: caddyfileHeader,
},
{
name: "containers with unsupported protocols and host mode ignored",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "5000/tcp"),
newContainer("10.210.0.3", "5000/udp"),
newContainer("10.210.0.4", "80:8080/tcp@host"),
containers: []store.ContainerRecord{
newContainerRecord(newContainer("10.210.0.2", "5000/tcp"), "mach1"),
newContainerRecord(newContainer("10.210.0.3", "5000/udp"), "mach1"),
newContainerRecord(newContainer("10.210.0.4", "80:8080/tcp@host"), "mach1"),
},
want: caddyfileHeader,
},
{
name: "restarting container ignored",
containers: []api.ServiceContainer{
newRestartingContainer("10.210.0.2", "app.example.com:8080/http"),
},
want: caddyfileHeader,
},
{
name: "stopped container ignored",
containers: []api.ServiceContainer{
newStoppedContainer("10.210.0.2", "app.example.com:8080/http"),
},
want: caddyfileHeader,
},
{
name: "mix of running, restarting, and stopped containers",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "app.example.com:8080/http"),
newRestartingContainer("10.210.0.3", "app.example.com:8080/http"),
newStoppedContainer("10.210.0.4", "app.example.com:8080/http"),
},
want: caddyfileHeader + `
http://app.example.com {
reverse_proxy {
to 10.210.0.2:8080
import common_proxy
}
log
}
`,
},
}
ctx := context.Background()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config, err := generator.Generate(tt.containers)
config, err := generator.Generate(ctx, tt.containers)
if tt.wantErr {
assert.Error(t, err)
@@ -198,3 +181,10 @@ http://app.example.com {
})
}
}
func newContainerRecord(ctr api.ServiceContainer, machineID string) store.ContainerRecord {
return store.ContainerRecord{
Container: ctr,
MachineID: machineID,
}
}
internal/machine/caddyconfig/controller.go
+33 -34
View File
@@ -14,8 +14,9 @@ import (
)
const (
CaddyGroup = "uncloud"
VerifyPath = "/.uncloud-verify"
CaddyServiceName = "caddy"
CaddyGroup = "uncloud"
VerifyPath = "/.uncloud-verify"
)
// Controller monitors container changes in the cluster store and generates a configuration file for Caddy reverse
@@ -37,32 +38,28 @@ func NewController(machineID, configDir, adminSock string, store *store.Store) (
return nil, fmt.Errorf("change owner of directory for Caddy configuration '%s': %w", configDir, err)
}
generator := &CaddyfileGenerator{
MachineID: machineID,
}
log := slog.With("component", "caddy-controller")
validator := NewCaddyAdminValidator(adminSock)
generator := NewCaddyfileGenerator(machineID, validator, log)
return &Controller{
machineID: machineID,
configDir: configDir,
generator: generator,
store: store,
log: slog.With("component", "caddy-controller"),
log: log,
}, nil
}
func (c *Controller) Run(ctx context.Context) error {
containerRecords, changes, err := c.store.SubscribeContainers(ctx)
containers, changes, err := c.store.SubscribeContainers(ctx)
if err != nil {
return fmt.Errorf("subscribe to container changes: %w", err)
}
c.log.Info("Subscribed to container changes in the cluster to generate Caddy configuration.")
containers, err := c.filterAvailableContainers(containerRecords)
if err != nil {
return fmt.Errorf("filter available containers: %w", err)
}
if err = c.generateCaddyfile(containers); err != nil {
containers = filterHealthyContainers(containers)
if err = c.generateCaddyfile(ctx, containers); err != nil {
return fmt.Errorf("generate Caddyfile configuration: %w", err)
}
if err = c.generateJSONConfig(containers); err != nil {
@@ -77,18 +74,14 @@ func (c *Controller) Run(ctx context.Context) error {
}
c.log.Info("Cluster containers changed, updating Caddy configuration.")
containerRecords, err = c.store.ListContainers(ctx, store.ListOptions{})
containers, err = c.store.ListContainers(ctx, store.ListOptions{})
if err != nil {
c.log.Error("Failed to list containers.", "err", err)
continue
}
containers, err = c.filterAvailableContainers(containerRecords)
if err != nil {
c.log.Error("Failed to filter available containers.", "err", err)
continue
}
containers = filterHealthyContainers(containers)
if err = c.generateCaddyfile(containers); err != nil {
if err = c.generateCaddyfile(ctx, containers); err != nil {
c.log.Error("Failed to generate Caddyfile configuration.", "err", err)
}
if err = c.generateJSONConfig(containers); err != nil {
@@ -102,21 +95,22 @@ func (c *Controller) Run(ctx context.Context) error {
}
}
// filterAvailableContainers filters out containers from this machine that are likely unavailable. The availability
// is determined by the cluster membership state of the machine that the container is running on.
// TODO: implement machine membership check using Corrossion Admin client.
func (c *Controller) filterAvailableContainers(
containerRecords []store.ContainerRecord,
) ([]api.ServiceContainer, error) {
containers := make([]api.ServiceContainer, len(containerRecords))
for i, cr := range containerRecords {
containers[i] = cr.Container
// filterHealthyContainers filters out containers that are not healthy.
// TODO: Filters out containers from this machine that are likely unavailable. The availability can be determined
// by the cluster membership state of the machine that the container is running on. Implement machine membership
// check using Corrossion Admin client.
func filterHealthyContainers(containers []store.ContainerRecord) []store.ContainerRecord {
healthy := make([]store.ContainerRecord, 0, len(containers))
for _, cr := range containers {
if cr.Container.Healthy() {
healthy = append(healthy, cr)
}
}
return containers, nil
return healthy
}
func (c *Controller) generateCaddyfile(containers []api.ServiceContainer) error {
caddyfile, err := c.generator.Generate(containers)
func (c *Controller) generateCaddyfile(ctx context.Context, containers []store.ContainerRecord) error {
caddyfile, err := c.generator.Generate(ctx, containers)
if err != nil {
return fmt.Errorf("generate Caddyfile: %w", err)
}
@@ -133,8 +127,13 @@ func (c *Controller) generateCaddyfile(containers []api.ServiceContainer) error
return nil
}
func (c *Controller) generateJSONConfig(containers []api.ServiceContainer) error {
config, err := GenerateJSONConfig(containers, c.machineID)
func (c *Controller) generateJSONConfig(containers []store.ContainerRecord) error {
serviceContainers := make([]api.ServiceContainer, 0, len(containers))
for i, cr := range containers {
serviceContainers[i] = cr.Container
}
config, err := GenerateJSONConfig(serviceContainers, c.machineID)
if err != nil {
return err
}
internal/machine/caddyconfig/jsonconfig_test.go
-74
View File
@@ -312,68 +312,6 @@ func TestGenerateJSONConfig(t *testing.T) {
want: configWithoutServices,
wantErr: false,
},
{
name: "restarting container ignored",
containers: []api.ServiceContainer{
newRestartingContainer("10.210.0.2", "app.example.com:8080/http"),
},
want: configWithoutServices,
wantErr: false,
},
{
name: "stopped container ignored",
containers: []api.ServiceContainer{
newStoppedContainer("10.210.0.2", "app.example.com:8080/http"),
},
want: configWithoutServices,
wantErr: false,
},
{
name: "mix of running, restarting, and stopped containers",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "app.example.com:8080/http"),
newRestartingContainer("10.210.0.3", "app.example.com:8080/http"),
newStoppedContainer("10.210.0.4", "app.example.com:8080/http"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [
{
"match": [{"host": ["app.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [{"dial": "10.210.0.2:8080"}]
}]
},
{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}
],
"logs": {}
},
"https": {
"listen": [":443"],
"logs": {}
}
}
}`,
wantErr: false,
},
}
for _, tt := range tests {
@@ -439,15 +377,3 @@ func newContainerWithoutNetwork(ports ...string) api.ServiceContainer {
},
}}}
}
func newRestartingContainer(ip string, ports ...string) api.ServiceContainer {
ctr := newContainer(ip, ports...)
ctr.Container.State.Restarting = true
return ctr
}
func newStoppedContainer(ip string, ports ...string) api.ServiceContainer {
ctr := newContainer(ip, ports...)
ctr.Container.State.Running = false
return ctr
}
internal/machine/caddyconfig/validator.go
+71
View File
@@ -0,0 +1,71 @@
package caddyconfig
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"strings"
"time"
"github.com/caddyserver/caddy/v2"
)
// CaddyAdminValidator validates Caddyfile via the Caddy admin API.
type CaddyAdminValidator struct {
socketPath string
client *http.Client
}
func NewCaddyAdminValidator(socketPath string) *CaddyAdminValidator {
return &CaddyAdminValidator{
socketPath: socketPath,
client: &http.Client{
Timeout: 5 * time.Second,
Transport: &http.Transport{
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
return net.Dial("unix", socketPath)
},
},
},
}
}
// Validate checks if the provided Caddyfile can be adapted to Caddy JSON config using the running Caddy instance via
// its admin API. It doesn't guarantee that the Caddyfile is actually valid and can be loaded. For example, a tls
// directive with a missing certificate will pass the adaptation but will fail when Caddy tries to load it.
// But this is the best we can do over the admin API.
// TODO: run 'docker exec caddy-container caddy validate' to do proper validation or implement a Caddy module that
// exposes a validation endpoint.
func (c *CaddyAdminValidator) Validate(ctx context.Context, caddyfile string) error {
// Bogus host is used so that http.NewRequest is happy but it doesn't matter since we're using a Unix socket.
req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/adapt", strings.NewReader(caddyfile))
if err != nil {
return fmt.Errorf("create adapt request: %w", err)
}
req.Header.Set("Content-Type", "text/caddyfile")
resp, err := c.client.Do(req)
if err != nil {
return fmt.Errorf("send adapt request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return nil
}
body, _ := io.ReadAll(resp.Body)
// If the response is a 400 Bad Request, try to parse the error message from it.
if resp.StatusCode == http.StatusBadRequest {
var apiError caddy.APIError
if err = json.Unmarshal(body, &apiError); err == nil {
return errors.New(apiError.Message)
}
}
return errors.New(string(body))
}
pkg/api/service.go
+8
View File
@@ -62,6 +62,14 @@ type ServiceSpec struct {
Volumes []VolumeSpec
}
// CaddyConfig returns the Caddy reverse proxy configuration for the service or an empty string if it's not defined.
func (s *ServiceSpec) CaddyConfig() string {
if s.Caddy == nil {
return ""
}
return strings.TrimSpace(s.Caddy.Config)
}
func (s *ServiceSpec) Volume(name string) (VolumeSpec, bool) {
for _, v := range s.Volumes {
if v.Name == name {