chore: load Caddy config via admin API instead of watching Caddyfile change on fs

internal/machine/caddyconfig/client.go

@@ -0,0 +1,124 @@
+package caddyconfig
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/caddyserver/caddy/v2"
+)
+
+// CaddyAdminClient is a client for interacting with the Caddy admin API over a Unix socket.
+type CaddyAdminClient struct {
+	socketPath string
+	client     *http.Client
+}
+
+func NewCaddyAdminClient(socketPath string) *CaddyAdminClient {
+	return &CaddyAdminClient{
+		socketPath: socketPath,
+		client: &http.Client{
+			Timeout: 5 * time.Second,
+			Transport: &http.Transport{
+				DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+					return net.Dial("unix", socketPath)
+				},
+			},
+		},
+	}
+}
+
+// Adapt converts a Caddyfile to JSON configuration without loading or running it.
+func (c *CaddyAdminClient) Adapt(ctx context.Context, caddyfile string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/adapt", strings.NewReader(caddyfile))
+	if err != nil {
+		return "", fmt.Errorf("create adapt request: %w", err)
+	}
+	req.Header.Set("Content-Type", "text/caddyfile")
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("send adapt request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("read response body: %w", err)
+	}
+
+	if resp.StatusCode == http.StatusOK {
+		// Parse the response body to extract the result field.
+		var msg struct {
+			Result json.RawMessage `json:"result"`
+		}
+		if err = json.Unmarshal(body, &msg); err != nil {
+			return "", fmt.Errorf("parse adapt response: %w", err)
+		}
+		return string(msg.Result), nil
+	}
+
+	// If the response is a 400 Bad Request, try to parse the error message from it.
+	if resp.StatusCode == http.StatusBadRequest {
+		var apiError caddy.APIError
+		if err = json.Unmarshal(body, &apiError); err == nil {
+			return "", errors.New(apiError.Message)
+		}
+	}
+
+	return "", errors.New(string(body))
+}
+
+// Load loads a Caddyfile configuration into the Caddy instance running on the machine.
+// Due to a Caddy bug (https://github.com/caddyserver/caddy/issues/7246), we first adapt the Caddyfile to JSON
+// and then load the JSON config to get proper error handling.
+func (c *CaddyAdminClient) Load(ctx context.Context, caddyfile string) error {
+	jsonConfig, err := c.Adapt(ctx, caddyfile)
+	if err != nil {
+		return fmt.Errorf("adapt Caddyfile to JSON config: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/load", strings.NewReader(jsonConfig))
+	if err != nil {
+		return fmt.Errorf("create load request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return fmt.Errorf("send load request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return nil
+	}
+
+	body, _ := io.ReadAll(resp.Body)
+	// If the response is a 400 Bad Request, try to parse the error message from it.
+	if resp.StatusCode == http.StatusBadRequest {
+		var apiError caddy.APIError
+		if err = json.Unmarshal(body, &apiError); err == nil {
+			return fmt.Errorf("caddy responded with error: %s", apiError.Message)
+		}
+	}
+
+	return fmt.Errorf("caddy responded with error: HTTP %d: %s", resp.StatusCode, string(body))
+}
+
+// Validate checks if the provided Caddyfile can be adapted to Caddy JSON config using the running Caddy instance via
+// its admin API. It doesn't guarantee that the Caddyfile is actually valid and can be loaded. For example, a tls
+// directive with a missing certificate will pass the adaptation but will fail when Caddy tries to load it.
+// But this is the best we can do over the admin API.
+// TODO: run 'docker exec caddy-container caddy validate' to do proper validation or implement a Caddy module that
+// exposes a validation endpoint.
+func (c *CaddyAdminClient) Validate(ctx context.Context, caddyfile string) error {
+	_, err := c.Adapt(ctx, caddyfile)
+	return err
+}

internal/machine/caddyconfig/controller.go

@@ -23,11 +23,12 @@ const (
 // proxy. The generated configuration allows Caddy to route external traffic to service containers across the internal
 // network.
 type Controller struct {
-	machineID string
+	machineID     string
-	configDir string
+	caddyfilePath string
-	generator *CaddyfileGenerator
+	generator     *CaddyfileGenerator
-	store     *store.Store
+	client        *CaddyAdminClient
-	log       *slog.Logger
+	store         *store.Store
+	log           *slog.Logger
 }
 
 func NewController(machineID, configDir, adminSock string, store *store.Store) (*Controller, error) {
@@ -39,15 +40,16 @@ func NewController(machineID, configDir, adminSock string, store *store.Store) (
 	}
 
 	log := slog.With("component", "caddy-controller")
-	validator := NewCaddyAdminValidator(adminSock)
+	client := NewCaddyAdminClient(adminSock)
-	generator := NewCaddyfileGenerator(machineID, validator, log)
+	generator := NewCaddyfileGenerator(machineID, client, log)
 
 	return &Controller{
-		machineID: machineID,
+		machineID:     machineID,
-		configDir: configDir,
+		caddyfilePath: filepath.Join(configDir, "Caddyfile"),
-		generator: generator,
+		generator:     generator,
-		store:     store,
+		client:        client,
-		log:       log,
+		store:         store,
+		log:           log,
 	}, nil
 }
 
@@ -59,11 +61,11 @@ func (c *Controller) Run(ctx context.Context) error {
 	c.log.Info("Subscribed to container changes in the cluster to generate Caddy configuration.")
 
 	containers = filterHealthyContainers(containers)
-	if err = c.generateCaddyfile(ctx, containers); err != nil {
+	c.generateAndLoadCaddyfile(ctx, containers)
-		return fmt.Errorf("generate Caddyfile configuration: %w", err)
+
-	}
+	// TODO: left for backward compatibility, remove later.
 	if err = c.generateJSONConfig(containers); err != nil {
-		return fmt.Errorf("generate Caddy JSON configuration: %w", err)
+		c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err)
 	}
 
 	for {
@@ -80,15 +82,12 @@ func (c *Controller) Run(ctx context.Context) error {
 				continue
 			}
 			containers = filterHealthyContainers(containers)
+			c.generateAndLoadCaddyfile(ctx, containers)
 
-			if err = c.generateCaddyfile(ctx, containers); err != nil {
+			// TODO: left for backward compatibility, remove later.
-				c.log.Error("Failed to generate Caddyfile configuration.", "err", err)
-			}
 			if err = c.generateJSONConfig(containers); err != nil {
-				c.log.Error("Failed to generate Caddy JSON configuration.", "err", err)
+				c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err)
 			}
-
-			c.log.Info("Updated Caddy configuration.", "dir", c.configDir)
 		case <-ctx.Done():
 			return nil
 		}
@@ -109,22 +108,35 @@ func filterHealthyContainers(containers []store.ContainerRecord) []store.Contain
 	return healthy
 }
 
-func (c *Controller) generateCaddyfile(ctx context.Context, containers []store.ContainerRecord) error {
+func (c *Controller) generateAndLoadCaddyfile(ctx context.Context, containers []store.ContainerRecord) {
+	caddyfile, err := c.generateCaddyfile(ctx, containers)
+	if err != nil {
+		c.log.Error("Failed to generate Caddyfile configuration.", "err", err)
+		return
+	}
+
+	if err = c.client.Load(ctx, caddyfile); err != nil {
+		c.log.Error("Failed to load new Caddy configuration into local Caddy instance.",
+			"err", err, "path", c.caddyfilePath)
+	} else {
+		c.log.Info("New Caddy configuration loaded into local Caddy instance.", "path", c.caddyfilePath)
+	}
+}
+
+func (c *Controller) generateCaddyfile(ctx context.Context, containers []store.ContainerRecord) (string, error) {
 	caddyfile, err := c.generator.Generate(ctx, containers)
 	if err != nil {
-		return fmt.Errorf("generate Caddyfile: %w", err)
+		return "", fmt.Errorf("generate Caddyfile: %w", err)
-	}
-	caddyfilePath := filepath.Join(c.configDir, "Caddyfile")
-
-	// TODO: use atomic file write to avoid partial loads on Caddy watch reload.
-	if err = os.WriteFile(caddyfilePath, []byte(caddyfile), 0o640); err != nil {
-		return fmt.Errorf("write Caddyfile to file '%s': %w", caddyfilePath, err)
-	}
-	if err = fs.Chown(caddyfilePath, "", CaddyGroup); err != nil {
-		return fmt.Errorf("change owner of Caddyfile '%s': %w", caddyfilePath, err)
 	}
 
-	return nil
+	if err = os.WriteFile(c.caddyfilePath, []byte(caddyfile), 0o640); err != nil {
+		return "", fmt.Errorf("write Caddyfile to file '%s': %w", c.caddyfilePath, err)
+	}
+	if err = fs.Chown(c.caddyfilePath, "", CaddyGroup); err != nil {
+		return "", fmt.Errorf("change owner of Caddyfile '%s': %w", c.caddyfilePath, err)
+	}
+
+	return caddyfile, nil
 }
 
 func (c *Controller) generateJSONConfig(containers []store.ContainerRecord) error {
@@ -142,7 +154,7 @@ func (c *Controller) generateJSONConfig(containers []store.ContainerRecord) erro
 	if err != nil {
 		return fmt.Errorf("marshal Caddy configuration: %w", err)
 	}
-	configPath := filepath.Join(c.configDir, "caddy.json")
+	configPath := filepath.Join(filepath.Dir(c.caddyfilePath), "caddy.json")
 
 	if err = os.WriteFile(configPath, configBytes, 0o640); err != nil {
 		return fmt.Errorf("write Caddy configuration to file '%s': %w", configPath, err)

internal/machine/caddyconfig/validator.go

@@ -1,71 +0,0 @@
-package caddyconfig
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/caddyserver/caddy/v2"
-)
-
-// CaddyAdminValidator validates Caddyfile via the Caddy admin API.
-type CaddyAdminValidator struct {
-	socketPath string
-	client     *http.Client
-}
-
-func NewCaddyAdminValidator(socketPath string) *CaddyAdminValidator {
-	return &CaddyAdminValidator{
-		socketPath: socketPath,
-		client: &http.Client{
-			Timeout: 5 * time.Second,
-			Transport: &http.Transport{
-				DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-					return net.Dial("unix", socketPath)
-				},
-			},
-		},
-	}
-}
-
-// Validate checks if the provided Caddyfile can be adapted to Caddy JSON config using the running Caddy instance via
-// its admin API. It doesn't guarantee that the Caddyfile is actually valid and can be loaded. For example, a tls
-// directive with a missing certificate will pass the adaptation but will fail when Caddy tries to load it.
-// But this is the best we can do over the admin API.
-// TODO: run 'docker exec caddy-container caddy validate' to do proper validation or implement a Caddy module that
-// exposes a validation endpoint.
-func (c *CaddyAdminValidator) Validate(ctx context.Context, caddyfile string) error {
-	// Bogus host is used so that http.NewRequest is happy but it doesn't matter since we're using a Unix socket.
-	req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/adapt", strings.NewReader(caddyfile))
-	if err != nil {
-		return fmt.Errorf("create adapt request: %w", err)
-	}
-	req.Header.Set("Content-Type", "text/caddyfile")
-
-	resp, err := c.client.Do(req)
-	if err != nil {
-		return fmt.Errorf("send adapt request: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusOK {
-		return nil
-	}
-
-	body, _ := io.ReadAll(resp.Body)
-	// If the response is a 400 Bad Request, try to parse the error message from it.
-	if resp.StatusCode == http.StatusBadRequest {
-		var apiError caddy.APIError
-		if err = json.Unmarshal(body, &apiError); err == nil {
-			return errors.New(apiError.Message)
-		}
-	}
-
-	return errors.New(string(body))
-}

pkg/client/caddy.go

@@ -35,7 +35,7 @@ func (cli *Client) NewCaddyDeployment(image, config string, placement api.Placem
 
 	spec := api.ServiceSpec{
 		Container: api.ContainerSpec{
-			Command: []string{"caddy", "run", "-c", "/config/Caddyfile", "--watch"},
+			Command: []string{"caddy", "run", "-c", "/config/Caddyfile"},
 			Env: map[string]string{
 				"CADDY_ADMIN": "unix//run/caddy/admin.sock",
 			},