diff --git a/internal/machine/caddyconfig/caddyfile.go b/internal/machine/caddyconfig/caddyfile.go index 053be3c7..95a17600 100644 --- a/internal/machine/caddyconfig/caddyfile.go +++ b/internal/machine/caddyconfig/caddyfile.go @@ -2,13 +2,18 @@ package caddyconfig import ( "bytes" + "cmp" + "context" "fmt" "log/slog" + "maps" "net" + "slices" "strconv" "strings" "text/template" + "github.com/psviderski/uncloud/internal/machine/store" "github.com/psviderski/uncloud/pkg/api" ) @@ -47,14 +52,26 @@ https://{{$hostname}} { // CaddyfileGenerator generates a Caddyfile configuration for the Caddy reverse proxy. type CaddyfileGenerator struct { - // MachineID is the unique identifier of the machine where the controller is running. - MachineID string - Validator CaddyfileValidator + // machineID is the unique identifier of the machine where the controller is running. + machineID string + validator CaddyfileValidator + log *slog.Logger } // CaddyfileValidator is an interface for validating Caddyfile configurations. type CaddyfileValidator interface { - Validate(caddyfile string) error + Validate(ctx context.Context, caddyfile string) error +} + +func NewCaddyfileGenerator(machineID string, validator CaddyfileValidator, log *slog.Logger) *CaddyfileGenerator { + if log == nil { + log = slog.Default() + } + return &CaddyfileGenerator{ + machineID: machineID, + validator: validator, + log: log, + } } // Generate creates a Caddyfile configuration based on the provided service containers. @@ -70,15 +87,87 @@ type CaddyfileValidator interface { // [service-a x-caddy] // ... // [service-z x-caddy] -func (g *CaddyfileGenerator) Generate(containers []api.ServiceContainer) (string, error) { - baseCaddyfile, err := g.generateBaseFromPorts(containers) +func (g *CaddyfileGenerator) Generate(ctx context.Context, records []store.ContainerRecord) (string, error) { + containers := make([]api.ServiceContainer, len(records)) + for i, cr := range records { + containers[i] = cr.Container + } + // Sort containers by service name and container ID to generate a stable Caddyfile. + slices.SortFunc(containers, func(a, b api.ServiceContainer) int { + return cmp.Or( + strings.Compare(a.ServiceName(), b.ServiceName()), + strings.Compare(a.ID, b.ID), + ) + }) + + caddyfile, err := g.generateBaseFromPorts(containers) if err != nil { return "", fmt.Errorf("generate base Caddyfile from service ports: %w", err) } - // TODO: Implement support for custom Caddy configs (x-caddy) in service specs. + // Find the 'caddy' service container on this machine. Use the most recent one if multiple exist. + var caddyCtr *api.ServiceContainer + for _, cr := range records { + if cr.MachineID == g.machineID && cr.Container.ServiceName() == CaddyServiceName && + (caddyCtr == nil || cr.Container.Created > caddyCtr.Created) { + caddyCtr = &cr.Container + } + } - return baseCaddyfile, nil + // If the caddy container is running on this machine and has a custom Caddy config, prepend it to the generated + // Caddyfile and validate it. + if caddyCtr != nil && caddyCtr.ServiceSpec.CaddyConfig() != "" { + // TODO: render the template actions in the Caddy config. + caddyfileCandidate := caddyCtr.ServiceSpec.CaddyConfig() + "\n\n" + caddyfile + + if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil { + g.log.Error("Custom Caddy config on the caddy container on this machine is invalid, skipping it.", + "service", caddyCtr.ServiceName(), "container", caddyCtr.ID, "err", err) + } else { + caddyfile = caddyfileCandidate + } + } + + // There could be multiple service containers for the same service with different custom Caddy configs, for example, + // if the service has been partially updated. The most recent container for each service defines the current custom + // Caddy config for that service. + latestServiceContainers := make(map[string]api.ServiceContainer, len(containers)) + for _, ctr := range containers { + if latest, ok := latestServiceContainers[ctr.ServiceName()]; ok { + if ctr.Created > latest.Created { + latestServiceContainers[ctr.ServiceName()] = ctr + } + } else { + latestServiceContainers[ctr.ServiceName()] = ctr + } + } + sortedServiceNames := slices.Sorted(maps.Keys(latestServiceContainers)) + + for _, serviceName := range sortedServiceNames { + // Skip the caddy container as we already processed it. + if serviceName == CaddyServiceName { + continue + } + + ctr := latestServiceContainers[serviceName] + if ctr.ServiceSpec.CaddyConfig() == "" { + continue + } + + // TODO: render the template actions in the Caddy config. + caddyfileCandidate := fmt.Sprintf("%s\n# Service: %s\n%s\n", + caddyfile, ctr.ServiceName(), ctr.ServiceSpec.CaddyConfig()) + + if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil { + g.log.Error("Custom Caddy config for service is invalid, skipping it.", + "service", ctr.ServiceName(), "err", err) + continue + } + + caddyfile = caddyfileCandidate + } + + return caddyfile, nil } func (g *CaddyfileGenerator) generateBaseFromPorts(containers []api.ServiceContainer) (string, error) { @@ -97,7 +186,7 @@ func (g *CaddyfileGenerator) generateBaseFromPorts(containers []api.ServiceConta HTTPSHostUpstreams map[string][]string }{ VerifyPath: VerifyPath, - VerifyResponse: g.MachineID, + VerifyResponse: g.machineID, HTTPHostUpstreams: httpHostUpstreams, HTTPSHostUpstreams: httpsHostUpstreams, } @@ -111,16 +200,12 @@ func (g *CaddyfileGenerator) generateBaseFromPorts(containers []api.ServiceConta } // httpUpstreamsFromContainers extracts upstreams for HTTP and HTTPS protocols from the published ports of the provided -// service containers. +// service containers. It's expected that all containers are healthy. func httpUpstreamsFromContainers(containers []api.ServiceContainer) (map[string][]string, map[string][]string) { // Maps hostnames to lists of upstreams (container IP:port pairs). httpHostUpstreams := make(map[string][]string) httpsHostUpstreams := make(map[string][]string) for _, ctr := range containers { - if !ctr.Healthy() { - continue - } - ip := ctr.UncloudNetworkIP() if !ip.IsValid() { // Container is not connected to the uncloud Docker network (could be host network). diff --git a/internal/machine/caddyconfig/caddyfile_test.go b/internal/machine/caddyconfig/caddyfile_test.go index ce60c9f7..7e171e1e 100644 --- a/internal/machine/caddyconfig/caddyfile_test.go +++ b/internal/machine/caddyconfig/caddyfile_test.go @@ -1,8 +1,10 @@ package caddyconfig import ( + "context" "testing" + "github.com/psviderski/uncloud/internal/machine/store" "github.com/psviderski/uncloud/pkg/api" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,25 +26,24 @@ func TestCaddyfileGenerator(t *testing.T) { } ` - generator := &CaddyfileGenerator{ - MachineID: "test-machine-id", - } + // TODO: mock validator + generator := NewCaddyfileGenerator("test-machine-id", nil, nil) tests := []struct { name string - containers []api.ServiceContainer + containers []store.ContainerRecord want string wantErr bool }{ { name: "empty containers", - containers: []api.ServiceContainer{}, + containers: []store.ContainerRecord{}, want: caddyfileHeader, }, { name: "HTTP container", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "app.example.com:8080/http"), + containers: []store.ContainerRecord{ + newContainerRecord(newContainer("10.210.0.2", "app.example.com:8080/http"), "mach1"), }, want: caddyfileHeader + ` http://app.example.com { @@ -56,9 +57,9 @@ http://app.example.com { }, { name: "load balancing multiple containers", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "app.example.com:8080/http"), - newContainer("10.210.0.3", "app.example.com:8080/http"), + containers: []store.ContainerRecord{ + newContainerRecord(newContainer("10.210.0.2", "app.example.com:8080/http"), "mach1"), + newContainerRecord(newContainer("10.210.0.3", "app.example.com:8080/http"), "mach1"), }, want: caddyfileHeader + ` http://app.example.com { @@ -72,8 +73,8 @@ http://app.example.com { }, { name: "HTTPS container", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "secure.example.com:8000/https"), + containers: []store.ContainerRecord{ + newContainerRecord(newContainer("10.210.0.2", "secure.example.com:8000/https"), "mach1"), }, want: caddyfileHeader + ` https://secure.example.com { @@ -87,20 +88,32 @@ https://secure.example.com { }, { name: "mixed HTTP and HTTPS", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", - "app.example.com:8080/http", - "web.example.com:8000/http"), - newContainer("10.210.0.3", - "app.example.com:8080/http", - "secure.example.com:8888/https"), - newContainer("10.210.0.4", - "web.example.com:8000/http", - "secure.example.com:8888/https"), - newContainer("10.210.0.5", - "app.example.com:8080/http", - "web.example.com:8000/http", - "secure.example.com:8888/https"), + containers: []store.ContainerRecord{ + newContainerRecord( + newContainer("10.210.0.2", + "app.example.com:8080/http", + "web.example.com:8000/http"), + "mach1", + ), + newContainerRecord( + newContainer("10.210.0.3", + "app.example.com:8080/http", + "secure.example.com:8888/https"), + "mach1", + ), + newContainerRecord( + newContainer("10.210.0.4", + "web.example.com:8000/http", + "secure.example.com:8888/https"), + "mach1", + ), + newContainerRecord( + newContainer("10.210.0.5", + "app.example.com:8080/http", + "web.example.com:8000/http", + "secure.example.com:8888/https"), + "mach1", + ), }, want: caddyfileHeader + ` http://app.example.com { @@ -130,63 +143,33 @@ https://secure.example.com { }, { name: "container without uncloud network ignored", - containers: []api.ServiceContainer{ - newContainerWithoutNetwork("ignored.example.com:8080/http"), + containers: []store.ContainerRecord{ + newContainerRecord(newContainerWithoutNetwork("ignored.example.com:8080/http"), "mach1"), }, want: caddyfileHeader, }, { name: "container with invalid port ignored", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "invalid-port"), + containers: []store.ContainerRecord{ + newContainerRecord(newContainer("10.210.0.2", "invalid-port"), "mach1"), }, want: caddyfileHeader, }, { name: "containers with unsupported protocols and host mode ignored", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "5000/tcp"), - newContainer("10.210.0.3", "5000/udp"), - newContainer("10.210.0.4", "80:8080/tcp@host"), + containers: []store.ContainerRecord{ + newContainerRecord(newContainer("10.210.0.2", "5000/tcp"), "mach1"), + newContainerRecord(newContainer("10.210.0.3", "5000/udp"), "mach1"), + newContainerRecord(newContainer("10.210.0.4", "80:8080/tcp@host"), "mach1"), }, want: caddyfileHeader, }, - { - name: "restarting container ignored", - containers: []api.ServiceContainer{ - newRestartingContainer("10.210.0.2", "app.example.com:8080/http"), - }, - want: caddyfileHeader, - }, - { - name: "stopped container ignored", - containers: []api.ServiceContainer{ - newStoppedContainer("10.210.0.2", "app.example.com:8080/http"), - }, - want: caddyfileHeader, - }, - { - name: "mix of running, restarting, and stopped containers", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "app.example.com:8080/http"), - newRestartingContainer("10.210.0.3", "app.example.com:8080/http"), - newStoppedContainer("10.210.0.4", "app.example.com:8080/http"), - }, - want: caddyfileHeader + ` -http://app.example.com { - reverse_proxy { - to 10.210.0.2:8080 - import common_proxy - } - log -} -`, - }, } + ctx := context.Background() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - config, err := generator.Generate(tt.containers) + config, err := generator.Generate(ctx, tt.containers) if tt.wantErr { assert.Error(t, err) @@ -198,3 +181,10 @@ http://app.example.com { }) } } + +func newContainerRecord(ctr api.ServiceContainer, machineID string) store.ContainerRecord { + return store.ContainerRecord{ + Container: ctr, + MachineID: machineID, + } +} diff --git a/internal/machine/caddyconfig/controller.go b/internal/machine/caddyconfig/controller.go index c77d0b36..c4053451 100644 --- a/internal/machine/caddyconfig/controller.go +++ b/internal/machine/caddyconfig/controller.go @@ -14,8 +14,9 @@ import ( ) const ( - CaddyGroup = "uncloud" - VerifyPath = "/.uncloud-verify" + CaddyServiceName = "caddy" + CaddyGroup = "uncloud" + VerifyPath = "/.uncloud-verify" ) // Controller monitors container changes in the cluster store and generates a configuration file for Caddy reverse @@ -37,32 +38,28 @@ func NewController(machineID, configDir, adminSock string, store *store.Store) ( return nil, fmt.Errorf("change owner of directory for Caddy configuration '%s': %w", configDir, err) } - generator := &CaddyfileGenerator{ - MachineID: machineID, - } + log := slog.With("component", "caddy-controller") + validator := NewCaddyAdminValidator(adminSock) + generator := NewCaddyfileGenerator(machineID, validator, log) return &Controller{ machineID: machineID, configDir: configDir, generator: generator, store: store, - log: slog.With("component", "caddy-controller"), + log: log, }, nil } func (c *Controller) Run(ctx context.Context) error { - containerRecords, changes, err := c.store.SubscribeContainers(ctx) + containers, changes, err := c.store.SubscribeContainers(ctx) if err != nil { return fmt.Errorf("subscribe to container changes: %w", err) } c.log.Info("Subscribed to container changes in the cluster to generate Caddy configuration.") - containers, err := c.filterAvailableContainers(containerRecords) - if err != nil { - return fmt.Errorf("filter available containers: %w", err) - } - - if err = c.generateCaddyfile(containers); err != nil { + containers = filterHealthyContainers(containers) + if err = c.generateCaddyfile(ctx, containers); err != nil { return fmt.Errorf("generate Caddyfile configuration: %w", err) } if err = c.generateJSONConfig(containers); err != nil { @@ -77,18 +74,14 @@ func (c *Controller) Run(ctx context.Context) error { } c.log.Info("Cluster containers changed, updating Caddy configuration.") - containerRecords, err = c.store.ListContainers(ctx, store.ListOptions{}) + containers, err = c.store.ListContainers(ctx, store.ListOptions{}) if err != nil { c.log.Error("Failed to list containers.", "err", err) continue } - containers, err = c.filterAvailableContainers(containerRecords) - if err != nil { - c.log.Error("Failed to filter available containers.", "err", err) - continue - } + containers = filterHealthyContainers(containers) - if err = c.generateCaddyfile(containers); err != nil { + if err = c.generateCaddyfile(ctx, containers); err != nil { c.log.Error("Failed to generate Caddyfile configuration.", "err", err) } if err = c.generateJSONConfig(containers); err != nil { @@ -102,21 +95,22 @@ func (c *Controller) Run(ctx context.Context) error { } } -// filterAvailableContainers filters out containers from this machine that are likely unavailable. The availability -// is determined by the cluster membership state of the machine that the container is running on. -// TODO: implement machine membership check using Corrossion Admin client. -func (c *Controller) filterAvailableContainers( - containerRecords []store.ContainerRecord, -) ([]api.ServiceContainer, error) { - containers := make([]api.ServiceContainer, len(containerRecords)) - for i, cr := range containerRecords { - containers[i] = cr.Container +// filterHealthyContainers filters out containers that are not healthy. +// TODO: Filters out containers from this machine that are likely unavailable. The availability can be determined +// by the cluster membership state of the machine that the container is running on. Implement machine membership +// check using Corrossion Admin client. +func filterHealthyContainers(containers []store.ContainerRecord) []store.ContainerRecord { + healthy := make([]store.ContainerRecord, 0, len(containers)) + for _, cr := range containers { + if cr.Container.Healthy() { + healthy = append(healthy, cr) + } } - return containers, nil + return healthy } -func (c *Controller) generateCaddyfile(containers []api.ServiceContainer) error { - caddyfile, err := c.generator.Generate(containers) +func (c *Controller) generateCaddyfile(ctx context.Context, containers []store.ContainerRecord) error { + caddyfile, err := c.generator.Generate(ctx, containers) if err != nil { return fmt.Errorf("generate Caddyfile: %w", err) } @@ -133,8 +127,13 @@ func (c *Controller) generateCaddyfile(containers []api.ServiceContainer) error return nil } -func (c *Controller) generateJSONConfig(containers []api.ServiceContainer) error { - config, err := GenerateJSONConfig(containers, c.machineID) +func (c *Controller) generateJSONConfig(containers []store.ContainerRecord) error { + serviceContainers := make([]api.ServiceContainer, 0, len(containers)) + for i, cr := range containers { + serviceContainers[i] = cr.Container + } + + config, err := GenerateJSONConfig(serviceContainers, c.machineID) if err != nil { return err } diff --git a/internal/machine/caddyconfig/jsonconfig_test.go b/internal/machine/caddyconfig/jsonconfig_test.go index a3c928d3..ab2969a9 100644 --- a/internal/machine/caddyconfig/jsonconfig_test.go +++ b/internal/machine/caddyconfig/jsonconfig_test.go @@ -312,68 +312,6 @@ func TestGenerateJSONConfig(t *testing.T) { want: configWithoutServices, wantErr: false, }, - { - name: "restarting container ignored", - containers: []api.ServiceContainer{ - newRestartingContainer("10.210.0.2", "app.example.com:8080/http"), - }, - want: configWithoutServices, - wantErr: false, - }, - { - name: "stopped container ignored", - containers: []api.ServiceContainer{ - newStoppedContainer("10.210.0.2", "app.example.com:8080/http"), - }, - want: configWithoutServices, - wantErr: false, - }, - { - name: "mix of running, restarting, and stopped containers", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "app.example.com:8080/http"), - newRestartingContainer("10.210.0.3", "app.example.com:8080/http"), - newStoppedContainer("10.210.0.4", "app.example.com:8080/http"), - }, - want: `{ - "servers": { - "http": { - "listen": [":80"], - "routes": [ - { - "match": [{"host": ["app.example.com"]}], - "handle": [{ - "handler": "reverse_proxy", - "health_checks": { - "passive": { - "fail_duration": 30000000000 - } - }, - "load_balancing": { - "retries": 3 - }, - "upstreams": [{"dial": "10.210.0.2:8080"}] - }] - }, - { - "match": [{"path": ["/.uncloud-verify"]}], - "handle": [{ - "body": "verification-response-body", - "handler": "static_response", - "status_code": 200 - }] - } - ], - "logs": {} - }, - "https": { - "listen": [":443"], - "logs": {} - } - } - }`, - wantErr: false, - }, } for _, tt := range tests { @@ -439,15 +377,3 @@ func newContainerWithoutNetwork(ports ...string) api.ServiceContainer { }, }}} } - -func newRestartingContainer(ip string, ports ...string) api.ServiceContainer { - ctr := newContainer(ip, ports...) - ctr.Container.State.Restarting = true - return ctr -} - -func newStoppedContainer(ip string, ports ...string) api.ServiceContainer { - ctr := newContainer(ip, ports...) - ctr.Container.State.Running = false - return ctr -} diff --git a/internal/machine/caddyconfig/validator.go b/internal/machine/caddyconfig/validator.go new file mode 100644 index 00000000..6e6d1c6f --- /dev/null +++ b/internal/machine/caddyconfig/validator.go @@ -0,0 +1,71 @@ +package caddyconfig + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" + + "github.com/caddyserver/caddy/v2" +) + +// CaddyAdminValidator validates Caddyfile via the Caddy admin API. +type CaddyAdminValidator struct { + socketPath string + client *http.Client +} + +func NewCaddyAdminValidator(socketPath string) *CaddyAdminValidator { + return &CaddyAdminValidator{ + socketPath: socketPath, + client: &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", socketPath) + }, + }, + }, + } +} + +// Validate checks if the provided Caddyfile can be adapted to Caddy JSON config using the running Caddy instance via +// its admin API. It doesn't guarantee that the Caddyfile is actually valid and can be loaded. For example, a tls +// directive with a missing certificate will pass the adaptation but will fail when Caddy tries to load it. +// But this is the best we can do over the admin API. +// TODO: run 'docker exec caddy-container caddy validate' to do proper validation or implement a Caddy module that +// exposes a validation endpoint. +func (c *CaddyAdminValidator) Validate(ctx context.Context, caddyfile string) error { + // Bogus host is used so that http.NewRequest is happy but it doesn't matter since we're using a Unix socket. + req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/adapt", strings.NewReader(caddyfile)) + if err != nil { + return fmt.Errorf("create adapt request: %w", err) + } + req.Header.Set("Content-Type", "text/caddyfile") + + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("send adapt request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + return nil + } + + body, _ := io.ReadAll(resp.Body) + // If the response is a 400 Bad Request, try to parse the error message from it. + if resp.StatusCode == http.StatusBadRequest { + var apiError caddy.APIError + if err = json.Unmarshal(body, &apiError); err == nil { + return errors.New(apiError.Message) + } + } + + return errors.New(string(body)) +} diff --git a/pkg/api/service.go b/pkg/api/service.go index cc52002e..545df3cc 100644 --- a/pkg/api/service.go +++ b/pkg/api/service.go @@ -62,6 +62,14 @@ type ServiceSpec struct { Volumes []VolumeSpec } +// CaddyConfig returns the Caddy reverse proxy configuration for the service or an empty string if it's not defined. +func (s *ServiceSpec) CaddyConfig() string { + if s.Caddy == nil { + return "" + } + return strings.TrimSpace(s.Caddy.Config) +} + func (s *ServiceSpec) Volume(name string) (VolumeSpec, bool) { for _, v := range s.Volumes { if v.Name == name {