From 5baa8087e548ef9fc3756228e3a776b80af17620 Mon Sep 17 00:00:00 2001 From: Pasha Sviderski Date: Tue, 9 Sep 2025 19:10:32 +1000 Subject: [PATCH] chore: load Caddy config via admin API instead of watching Caddyfile change on fs --- internal/machine/caddyconfig/client.go | 124 +++++++++++++++++++++ internal/machine/caddyconfig/controller.go | 82 ++++++++------ internal/machine/caddyconfig/validator.go | 71 ------------ pkg/client/caddy.go | 2 +- 4 files changed, 172 insertions(+), 107 deletions(-) create mode 100644 internal/machine/caddyconfig/client.go delete mode 100644 internal/machine/caddyconfig/validator.go diff --git a/internal/machine/caddyconfig/client.go b/internal/machine/caddyconfig/client.go new file mode 100644 index 00000000..8a3a4c4b --- /dev/null +++ b/internal/machine/caddyconfig/client.go @@ -0,0 +1,124 @@ +package caddyconfig + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" + + "github.com/caddyserver/caddy/v2" +) + +// CaddyAdminClient is a client for interacting with the Caddy admin API over a Unix socket. +type CaddyAdminClient struct { + socketPath string + client *http.Client +} + +func NewCaddyAdminClient(socketPath string) *CaddyAdminClient { + return &CaddyAdminClient{ + socketPath: socketPath, + client: &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", socketPath) + }, + }, + }, + } +} + +// Adapt converts a Caddyfile to JSON configuration without loading or running it. +func (c *CaddyAdminClient) Adapt(ctx context.Context, caddyfile string) (string, error) { + req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/adapt", strings.NewReader(caddyfile)) + if err != nil { + return "", fmt.Errorf("create adapt request: %w", err) + } + req.Header.Set("Content-Type", "text/caddyfile") + + resp, err := c.client.Do(req) + if err != nil { + return "", fmt.Errorf("send adapt request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("read response body: %w", err) + } + + if resp.StatusCode == http.StatusOK { + // Parse the response body to extract the result field. + var msg struct { + Result json.RawMessage `json:"result"` + } + if err = json.Unmarshal(body, &msg); err != nil { + return "", fmt.Errorf("parse adapt response: %w", err) + } + return string(msg.Result), nil + } + + // If the response is a 400 Bad Request, try to parse the error message from it. + if resp.StatusCode == http.StatusBadRequest { + var apiError caddy.APIError + if err = json.Unmarshal(body, &apiError); err == nil { + return "", errors.New(apiError.Message) + } + } + + return "", errors.New(string(body)) +} + +// Load loads a Caddyfile configuration into the Caddy instance running on the machine. +// Due to a Caddy bug (https://github.com/caddyserver/caddy/issues/7246), we first adapt the Caddyfile to JSON +// and then load the JSON config to get proper error handling. +func (c *CaddyAdminClient) Load(ctx context.Context, caddyfile string) error { + jsonConfig, err := c.Adapt(ctx, caddyfile) + if err != nil { + return fmt.Errorf("adapt Caddyfile to JSON config: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/load", strings.NewReader(jsonConfig)) + if err != nil { + return fmt.Errorf("create load request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("send load request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + return nil + } + + body, _ := io.ReadAll(resp.Body) + // If the response is a 400 Bad Request, try to parse the error message from it. + if resp.StatusCode == http.StatusBadRequest { + var apiError caddy.APIError + if err = json.Unmarshal(body, &apiError); err == nil { + return fmt.Errorf("caddy responded with error: %s", apiError.Message) + } + } + + return fmt.Errorf("caddy responded with error: HTTP %d: %s", resp.StatusCode, string(body)) +} + +// Validate checks if the provided Caddyfile can be adapted to Caddy JSON config using the running Caddy instance via +// its admin API. It doesn't guarantee that the Caddyfile is actually valid and can be loaded. For example, a tls +// directive with a missing certificate will pass the adaptation but will fail when Caddy tries to load it. +// But this is the best we can do over the admin API. +// TODO: run 'docker exec caddy-container caddy validate' to do proper validation or implement a Caddy module that +// exposes a validation endpoint. +func (c *CaddyAdminClient) Validate(ctx context.Context, caddyfile string) error { + _, err := c.Adapt(ctx, caddyfile) + return err +} diff --git a/internal/machine/caddyconfig/controller.go b/internal/machine/caddyconfig/controller.go index bc0bfe92..890f1ce5 100644 --- a/internal/machine/caddyconfig/controller.go +++ b/internal/machine/caddyconfig/controller.go @@ -23,11 +23,12 @@ const ( // proxy. The generated configuration allows Caddy to route external traffic to service containers across the internal // network. type Controller struct { - machineID string - configDir string - generator *CaddyfileGenerator - store *store.Store - log *slog.Logger + machineID string + caddyfilePath string + generator *CaddyfileGenerator + client *CaddyAdminClient + store *store.Store + log *slog.Logger } func NewController(machineID, configDir, adminSock string, store *store.Store) (*Controller, error) { @@ -39,15 +40,16 @@ func NewController(machineID, configDir, adminSock string, store *store.Store) ( } log := slog.With("component", "caddy-controller") - validator := NewCaddyAdminValidator(adminSock) - generator := NewCaddyfileGenerator(machineID, validator, log) + client := NewCaddyAdminClient(adminSock) + generator := NewCaddyfileGenerator(machineID, client, log) return &Controller{ - machineID: machineID, - configDir: configDir, - generator: generator, - store: store, - log: log, + machineID: machineID, + caddyfilePath: filepath.Join(configDir, "Caddyfile"), + generator: generator, + client: client, + store: store, + log: log, }, nil } @@ -59,11 +61,11 @@ func (c *Controller) Run(ctx context.Context) error { c.log.Info("Subscribed to container changes in the cluster to generate Caddy configuration.") containers = filterHealthyContainers(containers) - if err = c.generateCaddyfile(ctx, containers); err != nil { - return fmt.Errorf("generate Caddyfile configuration: %w", err) - } + c.generateAndLoadCaddyfile(ctx, containers) + + // TODO: left for backward compatibility, remove later. if err = c.generateJSONConfig(containers); err != nil { - return fmt.Errorf("generate Caddy JSON configuration: %w", err) + c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err) } for { @@ -80,15 +82,12 @@ func (c *Controller) Run(ctx context.Context) error { continue } containers = filterHealthyContainers(containers) + c.generateAndLoadCaddyfile(ctx, containers) - if err = c.generateCaddyfile(ctx, containers); err != nil { - c.log.Error("Failed to generate Caddyfile configuration.", "err", err) - } + // TODO: left for backward compatibility, remove later. if err = c.generateJSONConfig(containers); err != nil { - c.log.Error("Failed to generate Caddy JSON configuration.", "err", err) + c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err) } - - c.log.Info("Updated Caddy configuration.", "dir", c.configDir) case <-ctx.Done(): return nil } @@ -109,22 +108,35 @@ func filterHealthyContainers(containers []store.ContainerRecord) []store.Contain return healthy } -func (c *Controller) generateCaddyfile(ctx context.Context, containers []store.ContainerRecord) error { +func (c *Controller) generateAndLoadCaddyfile(ctx context.Context, containers []store.ContainerRecord) { + caddyfile, err := c.generateCaddyfile(ctx, containers) + if err != nil { + c.log.Error("Failed to generate Caddyfile configuration.", "err", err) + return + } + + if err = c.client.Load(ctx, caddyfile); err != nil { + c.log.Error("Failed to load new Caddy configuration into local Caddy instance.", + "err", err, "path", c.caddyfilePath) + } else { + c.log.Info("New Caddy configuration loaded into local Caddy instance.", "path", c.caddyfilePath) + } +} + +func (c *Controller) generateCaddyfile(ctx context.Context, containers []store.ContainerRecord) (string, error) { caddyfile, err := c.generator.Generate(ctx, containers) if err != nil { - return fmt.Errorf("generate Caddyfile: %w", err) - } - caddyfilePath := filepath.Join(c.configDir, "Caddyfile") - - // TODO: use atomic file write to avoid partial loads on Caddy watch reload. - if err = os.WriteFile(caddyfilePath, []byte(caddyfile), 0o640); err != nil { - return fmt.Errorf("write Caddyfile to file '%s': %w", caddyfilePath, err) - } - if err = fs.Chown(caddyfilePath, "", CaddyGroup); err != nil { - return fmt.Errorf("change owner of Caddyfile '%s': %w", caddyfilePath, err) + return "", fmt.Errorf("generate Caddyfile: %w", err) } - return nil + if err = os.WriteFile(c.caddyfilePath, []byte(caddyfile), 0o640); err != nil { + return "", fmt.Errorf("write Caddyfile to file '%s': %w", c.caddyfilePath, err) + } + if err = fs.Chown(c.caddyfilePath, "", CaddyGroup); err != nil { + return "", fmt.Errorf("change owner of Caddyfile '%s': %w", c.caddyfilePath, err) + } + + return caddyfile, nil } func (c *Controller) generateJSONConfig(containers []store.ContainerRecord) error { @@ -142,7 +154,7 @@ func (c *Controller) generateJSONConfig(containers []store.ContainerRecord) erro if err != nil { return fmt.Errorf("marshal Caddy configuration: %w", err) } - configPath := filepath.Join(c.configDir, "caddy.json") + configPath := filepath.Join(filepath.Dir(c.caddyfilePath), "caddy.json") if err = os.WriteFile(configPath, configBytes, 0o640); err != nil { return fmt.Errorf("write Caddy configuration to file '%s': %w", configPath, err) diff --git a/internal/machine/caddyconfig/validator.go b/internal/machine/caddyconfig/validator.go deleted file mode 100644 index 6e6d1c6f..00000000 --- a/internal/machine/caddyconfig/validator.go +++ /dev/null @@ -1,71 +0,0 @@ -package caddyconfig - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net" - "net/http" - "strings" - "time" - - "github.com/caddyserver/caddy/v2" -) - -// CaddyAdminValidator validates Caddyfile via the Caddy admin API. -type CaddyAdminValidator struct { - socketPath string - client *http.Client -} - -func NewCaddyAdminValidator(socketPath string) *CaddyAdminValidator { - return &CaddyAdminValidator{ - socketPath: socketPath, - client: &http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { - return net.Dial("unix", socketPath) - }, - }, - }, - } -} - -// Validate checks if the provided Caddyfile can be adapted to Caddy JSON config using the running Caddy instance via -// its admin API. It doesn't guarantee that the Caddyfile is actually valid and can be loaded. For example, a tls -// directive with a missing certificate will pass the adaptation but will fail when Caddy tries to load it. -// But this is the best we can do over the admin API. -// TODO: run 'docker exec caddy-container caddy validate' to do proper validation or implement a Caddy module that -// exposes a validation endpoint. -func (c *CaddyAdminValidator) Validate(ctx context.Context, caddyfile string) error { - // Bogus host is used so that http.NewRequest is happy but it doesn't matter since we're using a Unix socket. - req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/adapt", strings.NewReader(caddyfile)) - if err != nil { - return fmt.Errorf("create adapt request: %w", err) - } - req.Header.Set("Content-Type", "text/caddyfile") - - resp, err := c.client.Do(req) - if err != nil { - return fmt.Errorf("send adapt request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusOK { - return nil - } - - body, _ := io.ReadAll(resp.Body) - // If the response is a 400 Bad Request, try to parse the error message from it. - if resp.StatusCode == http.StatusBadRequest { - var apiError caddy.APIError - if err = json.Unmarshal(body, &apiError); err == nil { - return errors.New(apiError.Message) - } - } - - return errors.New(string(body)) -} diff --git a/pkg/client/caddy.go b/pkg/client/caddy.go index 2f3e0751..aebde0aa 100644 --- a/pkg/client/caddy.go +++ b/pkg/client/caddy.go @@ -35,7 +35,7 @@ func (cli *Client) NewCaddyDeployment(image, config string, placement api.Placem spec := api.ServiceSpec{ Container: api.ContainerSpec{ - Command: []string{"caddy", "run", "-c", "/config/Caddyfile", "--watch"}, + Command: []string{"caddy", "run", "-c", "/config/Caddyfile"}, Env: map[string]string{ "CADDY_ADMIN": "unix//run/caddy/admin.sock", },