feat: concatenate custom Caddy configs for services into final Caddyfile (no upstream interpolation)

internal/machine/caddyconfig/caddyfile.go

@@ -2,13 +2,18 @@ package caddyconfig
 
 import (
 	"bytes"
+	"cmp"
+	"context"
 	"fmt"
 	"log/slog"
+	"maps"
 	"net"
+	"slices"
 	"strconv"
 	"strings"
 	"text/template"
 
+	"github.com/psviderski/uncloud/internal/machine/store"
 	"github.com/psviderski/uncloud/pkg/api"
 )
 
@@ -47,14 +52,26 @@ https://{{$hostname}} {
 
 // CaddyfileGenerator generates a Caddyfile configuration for the Caddy reverse proxy.
 type CaddyfileGenerator struct {
-	// MachineID is the unique identifier of the machine where the controller is running.
+	// machineID is the unique identifier of the machine where the controller is running.
-	MachineID string
+	machineID string
-	Validator CaddyfileValidator
+	validator CaddyfileValidator
+	log       *slog.Logger
 }
 
 // CaddyfileValidator is an interface for validating Caddyfile configurations.
 type CaddyfileValidator interface {
-	Validate(caddyfile string) error
+	Validate(ctx context.Context, caddyfile string) error
+}
+
+func NewCaddyfileGenerator(machineID string, validator CaddyfileValidator, log *slog.Logger) *CaddyfileGenerator {
+	if log == nil {
+		log = slog.Default()
+	}
+	return &CaddyfileGenerator{
+		machineID: machineID,
+		validator: validator,
+		log:       log,
+	}
 }
 
 // Generate creates a Caddyfile configuration based on the provided service containers.
@@ -70,15 +87,87 @@ type CaddyfileValidator interface {
 //	[service-a x-caddy]
 //	...
 //	[service-z x-caddy]
-func (g *CaddyfileGenerator) Generate(containers []api.ServiceContainer) (string, error) {
+func (g *CaddyfileGenerator) Generate(ctx context.Context, records []store.ContainerRecord) (string, error) {
-	baseCaddyfile, err := g.generateBaseFromPorts(containers)
+	containers := make([]api.ServiceContainer, len(records))
+	for i, cr := range records {
+		containers[i] = cr.Container
+	}
+	// Sort containers by service name and container ID to generate a stable Caddyfile.
+	slices.SortFunc(containers, func(a, b api.ServiceContainer) int {
+		return cmp.Or(
+			strings.Compare(a.ServiceName(), b.ServiceName()),
+			strings.Compare(a.ID, b.ID),
+		)
+	})
+
+	caddyfile, err := g.generateBaseFromPorts(containers)
 	if err != nil {
 		return "", fmt.Errorf("generate base Caddyfile from service ports: %w", err)
 	}
 
-	// TODO: Implement support for custom Caddy configs (x-caddy) in service specs.
+	// Find the 'caddy' service container on this machine. Use the most recent one if multiple exist.
+	var caddyCtr *api.ServiceContainer
+	for _, cr := range records {
+		if cr.MachineID == g.machineID && cr.Container.ServiceName() == CaddyServiceName &&
+			(caddyCtr == nil || cr.Container.Created > caddyCtr.Created) {
+			caddyCtr = &cr.Container
+		}
+	}
 
-	return baseCaddyfile, nil
+	// If the caddy container is running on this machine and has a custom Caddy config, prepend it to the generated
+	// Caddyfile and validate it.
+	if caddyCtr != nil && caddyCtr.ServiceSpec.CaddyConfig() != "" {
+		// TODO: render the template actions in the Caddy config.
+		caddyfileCandidate := caddyCtr.ServiceSpec.CaddyConfig() + "\n\n" + caddyfile
+
+		if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil {
+			g.log.Error("Custom Caddy config on the caddy container on this machine is invalid, skipping it.",
+				"service", caddyCtr.ServiceName(), "container", caddyCtr.ID, "err", err)
+		} else {
+			caddyfile = caddyfileCandidate
+		}
+	}
+
+	// There could be multiple service containers for the same service with different custom Caddy configs, for example,
+	// if the service has been partially updated. The most recent container for each service defines the current custom
+	// Caddy config for that service.
+	latestServiceContainers := make(map[string]api.ServiceContainer, len(containers))
+	for _, ctr := range containers {
+		if latest, ok := latestServiceContainers[ctr.ServiceName()]; ok {
+			if ctr.Created > latest.Created {
+				latestServiceContainers[ctr.ServiceName()] = ctr
+			}
+		} else {
+			latestServiceContainers[ctr.ServiceName()] = ctr
+		}
+	}
+	sortedServiceNames := slices.Sorted(maps.Keys(latestServiceContainers))
+
+	for _, serviceName := range sortedServiceNames {
+		// Skip the caddy container as we already processed it.
+		if serviceName == CaddyServiceName {
+			continue
+		}
+
+		ctr := latestServiceContainers[serviceName]
+		if ctr.ServiceSpec.CaddyConfig() == "" {
+			continue
+		}
+
+		// TODO: render the template actions in the Caddy config.
+		caddyfileCandidate := fmt.Sprintf("%s\n# Service: %s\n%s\n",
+			caddyfile, ctr.ServiceName(), ctr.ServiceSpec.CaddyConfig())
+
+		if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil {
+			g.log.Error("Custom Caddy config for service is invalid, skipping it.",
+				"service", ctr.ServiceName(), "err", err)
+			continue
+		}
+
+		caddyfile = caddyfileCandidate
+	}
+
+	return caddyfile, nil
 }
 
 func (g *CaddyfileGenerator) generateBaseFromPorts(containers []api.ServiceContainer) (string, error) {
@@ -97,7 +186,7 @@ func (g *CaddyfileGenerator) generateBaseFromPorts(containers []api.ServiceConta
 		HTTPSHostUpstreams map[string][]string
 	}{
 		VerifyPath:         VerifyPath,
-		VerifyResponse:     g.MachineID,
+		VerifyResponse:     g.machineID,
 		HTTPHostUpstreams:  httpHostUpstreams,
 		HTTPSHostUpstreams: httpsHostUpstreams,
 	}
@@ -111,16 +200,12 @@ func (g *CaddyfileGenerator) generateBaseFromPorts(containers []api.ServiceConta
 }
 
 // httpUpstreamsFromContainers extracts upstreams for HTTP and HTTPS protocols from the published ports of the provided
-// service containers.
+// service containers. It's expected that all containers are healthy.
 func httpUpstreamsFromContainers(containers []api.ServiceContainer) (map[string][]string, map[string][]string) {
 	// Maps hostnames to lists of upstreams (container IP:port pairs).
 	httpHostUpstreams := make(map[string][]string)
 	httpsHostUpstreams := make(map[string][]string)
 	for _, ctr := range containers {
-		if !ctr.Healthy() {
-			continue
-		}
-
 		ip := ctr.UncloudNetworkIP()
 		if !ip.IsValid() {
 			// Container is not connected to the uncloud Docker network (could be host network).

internal/machine/caddyconfig/caddyfile_test.go

@@ -1,8 +1,10 @@
 package caddyconfig
 
 import (
+	"context"
 	"testing"
 
+	"github.com/psviderski/uncloud/internal/machine/store"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -24,25 +26,24 @@ func TestCaddyfileGenerator(t *testing.T) {
 }
 `
 
-	generator := &CaddyfileGenerator{
+	// TODO: mock validator
-		MachineID: "test-machine-id",
+	generator := NewCaddyfileGenerator("test-machine-id", nil, nil)
-	}
 
 	tests := []struct {
 		name       string
-		containers []api.ServiceContainer
+		containers []store.ContainerRecord
 		want       string
 		wantErr    bool
 	}{
 		{
 			name:       "empty containers",
-			containers: []api.ServiceContainer{},
+			containers: []store.ContainerRecord{},
 			want:       caddyfileHeader,
 		},
 		{
 			name: "HTTP container",
-			containers: []api.ServiceContainer{
+			containers: []store.ContainerRecord{
-				newContainer("10.210.0.2", "app.example.com:8080/http"),
+				newContainerRecord(newContainer("10.210.0.2", "app.example.com:8080/http"), "mach1"),
 			},
 			want: caddyfileHeader + `
 http://app.example.com {
@@ -56,9 +57,9 @@ http://app.example.com {
 		},
 		{
 			name: "load balancing multiple containers",
-			containers: []api.ServiceContainer{
+			containers: []store.ContainerRecord{
-				newContainer("10.210.0.2", "app.example.com:8080/http"),
+				newContainerRecord(newContainer("10.210.0.2", "app.example.com:8080/http"), "mach1"),
-				newContainer("10.210.0.3", "app.example.com:8080/http"),
+				newContainerRecord(newContainer("10.210.0.3", "app.example.com:8080/http"), "mach1"),
 			},
 			want: caddyfileHeader + `
 http://app.example.com {
@@ -72,8 +73,8 @@ http://app.example.com {
 		},
 		{
 			name: "HTTPS container",
-			containers: []api.ServiceContainer{
+			containers: []store.ContainerRecord{
-				newContainer("10.210.0.2", "secure.example.com:8000/https"),
+				newContainerRecord(newContainer("10.210.0.2", "secure.example.com:8000/https"), "mach1"),
 			},
 			want: caddyfileHeader + `
 https://secure.example.com {
@@ -87,20 +88,32 @@ https://secure.example.com {
 		},
 		{
 			name: "mixed HTTP and HTTPS",
-			containers: []api.ServiceContainer{
+			containers: []store.ContainerRecord{
+				newContainerRecord(
 					newContainer("10.210.0.2",
 						"app.example.com:8080/http",
 						"web.example.com:8000/http"),
+					"mach1",
+				),
+				newContainerRecord(
 					newContainer("10.210.0.3",
 						"app.example.com:8080/http",
 						"secure.example.com:8888/https"),
+					"mach1",
+				),
+				newContainerRecord(
 					newContainer("10.210.0.4",
 						"web.example.com:8000/http",
 						"secure.example.com:8888/https"),
+					"mach1",
+				),
+				newContainerRecord(
 					newContainer("10.210.0.5",
 						"app.example.com:8080/http",
 						"web.example.com:8000/http",
 						"secure.example.com:8888/https"),
+					"mach1",
+				),
 			},
 			want: caddyfileHeader + `
 http://app.example.com {
@@ -130,63 +143,33 @@ https://secure.example.com {
 		},
 		{
 			name: "container without uncloud network ignored",
-			containers: []api.ServiceContainer{
+			containers: []store.ContainerRecord{
-				newContainerWithoutNetwork("ignored.example.com:8080/http"),
+				newContainerRecord(newContainerWithoutNetwork("ignored.example.com:8080/http"), "mach1"),
 			},
 			want: caddyfileHeader,
 		},
 		{
 			name: "container with invalid port ignored",
-			containers: []api.ServiceContainer{
+			containers: []store.ContainerRecord{
-				newContainer("10.210.0.2", "invalid-port"),
+				newContainerRecord(newContainer("10.210.0.2", "invalid-port"), "mach1"),
 			},
 			want: caddyfileHeader,
 		},
 		{
 			name: "containers with unsupported protocols and host mode ignored",
-			containers: []api.ServiceContainer{
+			containers: []store.ContainerRecord{
-				newContainer("10.210.0.2", "5000/tcp"),
+				newContainerRecord(newContainer("10.210.0.2", "5000/tcp"), "mach1"),
-				newContainer("10.210.0.3", "5000/udp"),
+				newContainerRecord(newContainer("10.210.0.3", "5000/udp"), "mach1"),
-				newContainer("10.210.0.4", "80:8080/tcp@host"),
+				newContainerRecord(newContainer("10.210.0.4", "80:8080/tcp@host"), "mach1"),
 			},
 			want: caddyfileHeader,
 		},
-		{
-			name: "restarting container ignored",
-			containers: []api.ServiceContainer{
-				newRestartingContainer("10.210.0.2", "app.example.com:8080/http"),
-			},
-			want: caddyfileHeader,
-		},
-		{
-			name: "stopped container ignored",
-			containers: []api.ServiceContainer{
-				newStoppedContainer("10.210.0.2", "app.example.com:8080/http"),
-			},
-			want: caddyfileHeader,
-		},
-		{
-			name: "mix of running, restarting, and stopped containers",
-			containers: []api.ServiceContainer{
-				newContainer("10.210.0.2", "app.example.com:8080/http"),
-				newRestartingContainer("10.210.0.3", "app.example.com:8080/http"),
-				newStoppedContainer("10.210.0.4", "app.example.com:8080/http"),
-			},
-			want: caddyfileHeader + `
-http://app.example.com {
-	reverse_proxy {
-		to 10.210.0.2:8080
-		import common_proxy
-	}
-	log
-}
-`,
-		},
 	}
 
+	ctx := context.Background()
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			config, err := generator.Generate(tt.containers)
+			config, err := generator.Generate(ctx, tt.containers)
 
 			if tt.wantErr {
 				assert.Error(t, err)
@@ -198,3 +181,10 @@ http://app.example.com {
 		})
 	}
 }
+
+func newContainerRecord(ctr api.ServiceContainer, machineID string) store.ContainerRecord {
+	return store.ContainerRecord{
+		Container: ctr,
+		MachineID: machineID,
+	}
+}

internal/machine/caddyconfig/controller.go

@@ -14,6 +14,7 @@ import (
 )
 
 const (
+	CaddyServiceName = "caddy"
 	CaddyGroup       = "uncloud"
 	VerifyPath       = "/.uncloud-verify"
 )
@@ -37,32 +38,28 @@ func NewController(machineID, configDir, adminSock string, store *store.Store) (
 		return nil, fmt.Errorf("change owner of directory for Caddy configuration '%s': %w", configDir, err)
 	}
 
-	generator := &CaddyfileGenerator{
+	log := slog.With("component", "caddy-controller")
-		MachineID: machineID,
+	validator := NewCaddyAdminValidator(adminSock)
-	}
+	generator := NewCaddyfileGenerator(machineID, validator, log)
 
 	return &Controller{
 		machineID: machineID,
 		configDir: configDir,
 		generator: generator,
 		store:     store,
-		log:       slog.With("component", "caddy-controller"),
+		log:       log,
 	}, nil
 }
 
 func (c *Controller) Run(ctx context.Context) error {
-	containerRecords, changes, err := c.store.SubscribeContainers(ctx)
+	containers, changes, err := c.store.SubscribeContainers(ctx)
 	if err != nil {
 		return fmt.Errorf("subscribe to container changes: %w", err)
 	}
 	c.log.Info("Subscribed to container changes in the cluster to generate Caddy configuration.")
 
-	containers, err := c.filterAvailableContainers(containerRecords)
+	containers = filterHealthyContainers(containers)
-	if err != nil {
+	if err = c.generateCaddyfile(ctx, containers); err != nil {
-		return fmt.Errorf("filter available containers: %w", err)
-	}
-
-	if err = c.generateCaddyfile(containers); err != nil {
 		return fmt.Errorf("generate Caddyfile configuration: %w", err)
 	}
 	if err = c.generateJSONConfig(containers); err != nil {
@@ -77,18 +74,14 @@ func (c *Controller) Run(ctx context.Context) error {
 			}
 			c.log.Info("Cluster containers changed, updating Caddy configuration.")
 
-			containerRecords, err = c.store.ListContainers(ctx, store.ListOptions{})
+			containers, err = c.store.ListContainers(ctx, store.ListOptions{})
 			if err != nil {
 				c.log.Error("Failed to list containers.", "err", err)
 				continue
 			}
-			containers, err = c.filterAvailableContainers(containerRecords)
+			containers = filterHealthyContainers(containers)
-			if err != nil {
-				c.log.Error("Failed to filter available containers.", "err", err)
-				continue
-			}
 
-			if err = c.generateCaddyfile(containers); err != nil {
+			if err = c.generateCaddyfile(ctx, containers); err != nil {
 				c.log.Error("Failed to generate Caddyfile configuration.", "err", err)
 			}
 			if err = c.generateJSONConfig(containers); err != nil {
@@ -102,21 +95,22 @@ func (c *Controller) Run(ctx context.Context) error {
 	}
 }
 
-// filterAvailableContainers filters out containers from this machine that are likely unavailable. The availability
+// filterHealthyContainers filters out containers that are not healthy.
-// is determined by the cluster membership state of the machine that the container is running on.
+// TODO: Filters out containers from this machine that are likely unavailable. The availability can be determined
-// TODO: implement machine membership check using Corrossion Admin client.
+// by the cluster membership state of the machine that the container is running on. Implement machine membership
-func (c *Controller) filterAvailableContainers(
+// check using Corrossion Admin client.
-	containerRecords []store.ContainerRecord,
+func filterHealthyContainers(containers []store.ContainerRecord) []store.ContainerRecord {
-) ([]api.ServiceContainer, error) {
+	healthy := make([]store.ContainerRecord, 0, len(containers))
-	containers := make([]api.ServiceContainer, len(containerRecords))
+	for _, cr := range containers {
-	for i, cr := range containerRecords {
+		if cr.Container.Healthy() {
-		containers[i] = cr.Container
+			healthy = append(healthy, cr)
 		}
-	return containers, nil
+	}
+	return healthy
 }
 
-func (c *Controller) generateCaddyfile(containers []api.ServiceContainer) error {
+func (c *Controller) generateCaddyfile(ctx context.Context, containers []store.ContainerRecord) error {
-	caddyfile, err := c.generator.Generate(containers)
+	caddyfile, err := c.generator.Generate(ctx, containers)
 	if err != nil {
 		return fmt.Errorf("generate Caddyfile: %w", err)
 	}
@@ -133,8 +127,13 @@ func (c *Controller) generateCaddyfile(containers []api.ServiceContainer) error
 	return nil
 }
 
-func (c *Controller) generateJSONConfig(containers []api.ServiceContainer) error {
+func (c *Controller) generateJSONConfig(containers []store.ContainerRecord) error {
-	config, err := GenerateJSONConfig(containers, c.machineID)
+	serviceContainers := make([]api.ServiceContainer, 0, len(containers))
+	for i, cr := range containers {
+		serviceContainers[i] = cr.Container
+	}
+
+	config, err := GenerateJSONConfig(serviceContainers, c.machineID)
 	if err != nil {
 		return err
 	}

internal/machine/caddyconfig/jsonconfig_test.go

@@ -312,68 +312,6 @@ func TestGenerateJSONConfig(t *testing.T) {
 			want:    configWithoutServices,
 			wantErr: false,
 		},
-		{
-			name: "restarting container ignored",
-			containers: []api.ServiceContainer{
-				newRestartingContainer("10.210.0.2", "app.example.com:8080/http"),
-			},
-			want:    configWithoutServices,
-			wantErr: false,
-		},
-		{
-			name: "stopped container ignored",
-			containers: []api.ServiceContainer{
-				newStoppedContainer("10.210.0.2", "app.example.com:8080/http"),
-			},
-			want:    configWithoutServices,
-			wantErr: false,
-		},
-		{
-			name: "mix of running, restarting, and stopped containers",
-			containers: []api.ServiceContainer{
-				newContainer("10.210.0.2", "app.example.com:8080/http"),
-				newRestartingContainer("10.210.0.3", "app.example.com:8080/http"),
-				newStoppedContainer("10.210.0.4", "app.example.com:8080/http"),
-			},
-			want: `{
-                "servers": {
-                    "http": {
-                        "listen": [":80"],
-                        "routes": [
-                            {
-                                "match": [{"host": ["app.example.com"]}],
-                                "handle": [{
-                                    "handler": "reverse_proxy",
-									"health_checks": {
-                                        "passive": {
-                                            "fail_duration": 30000000000
-                                        }
-                                    },
-                                    "load_balancing": {
-                                        "retries": 3
-                                    },
-                                    "upstreams": [{"dial": "10.210.0.2:8080"}]
-                                }]
-                            },
-                            {
-                                "match": [{"path": ["/.uncloud-verify"]}],
-                                "handle": [{
-                                    "body": "verification-response-body",
-                                    "handler": "static_response",
-                                    "status_code": 200
-                                }]
-                            }
-                        ],
-						"logs": {}
-                    },
-                    "https": {
-                        "listen": [":443"],
-						"logs": {}
-                    }
-                }
-            }`,
-			wantErr: false,
-		},
 	}
 
 	for _, tt := range tests {
@@ -439,15 +377,3 @@ func newContainerWithoutNetwork(ports ...string) api.ServiceContainer {
 		},
 	}}}
 }
-
-func newRestartingContainer(ip string, ports ...string) api.ServiceContainer {
-	ctr := newContainer(ip, ports...)
-	ctr.Container.State.Restarting = true
-	return ctr
-}
-
-func newStoppedContainer(ip string, ports ...string) api.ServiceContainer {
-	ctr := newContainer(ip, ports...)
-	ctr.Container.State.Running = false
-	return ctr
-}

internal/machine/caddyconfig/validator.go

@@ -0,0 +1,71 @@
+package caddyconfig
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/caddyserver/caddy/v2"
+)
+
+// CaddyAdminValidator validates Caddyfile via the Caddy admin API.
+type CaddyAdminValidator struct {
+	socketPath string
+	client     *http.Client
+}
+
+func NewCaddyAdminValidator(socketPath string) *CaddyAdminValidator {
+	return &CaddyAdminValidator{
+		socketPath: socketPath,
+		client: &http.Client{
+			Timeout: 5 * time.Second,
+			Transport: &http.Transport{
+				DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+					return net.Dial("unix", socketPath)
+				},
+			},
+		},
+	}
+}
+
+// Validate checks if the provided Caddyfile can be adapted to Caddy JSON config using the running Caddy instance via
+// its admin API. It doesn't guarantee that the Caddyfile is actually valid and can be loaded. For example, a tls
+// directive with a missing certificate will pass the adaptation but will fail when Caddy tries to load it.
+// But this is the best we can do over the admin API.
+// TODO: run 'docker exec caddy-container caddy validate' to do proper validation or implement a Caddy module that
+// exposes a validation endpoint.
+func (c *CaddyAdminValidator) Validate(ctx context.Context, caddyfile string) error {
+	// Bogus host is used so that http.NewRequest is happy but it doesn't matter since we're using a Unix socket.
+	req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/adapt", strings.NewReader(caddyfile))
+	if err != nil {
+		return fmt.Errorf("create adapt request: %w", err)
+	}
+	req.Header.Set("Content-Type", "text/caddyfile")
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return fmt.Errorf("send adapt request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return nil
+	}
+
+	body, _ := io.ReadAll(resp.Body)
+	// If the response is a 400 Bad Request, try to parse the error message from it.
+	if resp.StatusCode == http.StatusBadRequest {
+		var apiError caddy.APIError
+		if err = json.Unmarshal(body, &apiError); err == nil {
+			return errors.New(apiError.Message)
+		}
+	}
+
+	return errors.New(string(body))
+}

pkg/api/service.go

@@ -62,6 +62,14 @@ type ServiceSpec struct {
 	Volumes []VolumeSpec
 }
 
+// CaddyConfig returns the Caddy reverse proxy configuration for the service or an empty string if it's not defined.
+func (s *ServiceSpec) CaddyConfig() string {
+	if s.Caddy == nil {
+		return ""
+	}
+	return strings.TrimSpace(s.Caddy.Config)
+}
+
 func (s *ServiceSpec) Volume(name string) (VolumeSpec, bool) {
 	for _, v := range s.Volumes {
 		if v.Name == name {