uncloud/internal/machine/caddyconfig/controller.go

package caddyconfig

import (
	"context"
	"encoding/json"
	"fmt"
	"log/slog"
	"net/netip"
	"os"
	"path/filepath"
	"slices"
	"strings"

	"github.com/psviderski/uncloud/internal/fs"
	"github.com/psviderski/uncloud/internal/machine/store"
	"github.com/psviderski/uncloud/pkg/api"
)

const (
	CaddyServiceName = "caddy"
	CaddyGroup       = "uncloud"
	VerifyPath       = "/.uncloud-verify"
)

// Controller monitors container changes in the cluster store and generates a configuration file for Caddy reverse
// proxy. The generated configuration allows Caddy to route external traffic to service containers across the internal
// network.
type Controller struct {
	machineID     string
	caddyfilePath string
	generator     *CaddyfileGenerator
	client        *CaddyAdminClient
	store         *store.Store
	log           *slog.Logger
	// lastFingerprint caches the fingerprint of the containers used to generate the latest successfully loaded
	// Caddyfile. nil means it hasn't been loaded yet or the last load failed.
	lastFingerprint []containerFingerprint
	// lastCaddyfile caches the last generated Caddyfile.
	lastCaddyfile string
}

// containerFingerprint is the subset of container data that the Caddyfile generator depends on.
// Comparing fingerprints lets the controller skip no-op regenerations.
type containerFingerprint struct {
	ID          string
	IP          netip.Addr
	Ports       []api.PortSpec
	CaddyConfig string
}

// Equal returns whether two fingerprints describe the same container input to the Caddyfile generator.
func (f containerFingerprint) Equal(other containerFingerprint) bool {
	return f.ID == other.ID &&
		f.IP == other.IP &&
		api.PortsEqual(f.Ports, other.Ports) &&
		f.CaddyConfig == other.CaddyConfig
}

func NewController(machineID, configDir, adminSock string, store *store.Store) (*Controller, error) {
	if err := os.MkdirAll(configDir, 0o750); err != nil {
		return nil, fmt.Errorf("create directory for Caddy configuration '%s': %w", configDir, err)
	}
	if err := fs.Chown(configDir, "", CaddyGroup); err != nil {
		return nil, fmt.Errorf("change owner of directory for Caddy configuration '%s': %w", configDir, err)
	}

	log := slog.With("component", "caddy-controller")
	client := NewCaddyAdminClient(adminSock)

	// generator is initialised by Run() once the machine name is resolved from the store.
	return &Controller{
		machineID:     machineID,
		caddyfilePath: filepath.Join(configDir, "Caddyfile"),
		client:        client,
		store:         store,
		log:           log,
	}, nil
}

func (c *Controller) Run(ctx context.Context) error {
	// Default the machine name to the machine ID so the Caddyfile header still carries a stable identifier if
	// the store lookup fails.
	machineName := c.machineID
	if m, err := c.store.GetMachine(ctx, c.machineID); err != nil {
		c.log.Error("Failed to get machine from store, Caddy configuration will use machine ID as the name.",
			"machine_id", c.machineID, "err", err)
	} else {
		machineName = m.Name
	}
	c.generator = NewCaddyfileGenerator(c.machineID, machineName, c.client, c.log)

	containers, changes, err := c.store.SubscribeContainers(ctx)
	if err != nil {
		return fmt.Errorf("subscribe to container changes: %w", err)
	}
	c.log.Info("Subscribed to container changes in the cluster to generate Caddy configuration.")

	containers = filterHealthyContainers(containers)
	c.generateAndLoadCaddyfile(ctx, containers)

	// TODO: left for backward compatibility, remove later.
	if err = c.generateJSONConfig(containers); err != nil {
		c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err)
	}

	for {
		select {
		case _, ok := <-changes:
			if !ok {
				return fmt.Errorf("containers subscription failed")
			}
			c.log.Debug("Cluster containers changed, regenerating Caddy configuration.")

			containers, err = c.store.ListContainers(ctx, store.ListOptions{})
			if err != nil {
				c.log.Error("Failed to list containers.", "err", err)
				continue
			}
			containers = filterHealthyContainers(containers)
			c.generateAndLoadCaddyfile(ctx, containers)

			// TODO: left for backward compatibility, remove later.
			if err = c.generateJSONConfig(containers); err != nil {
				c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err)
			}
		case <-ctx.Done():
			return nil
		}
	}
}

// filterHealthyContainers filters out unhealthy and hook containers.
// TODO: Filters out containers from this machine that are likely unavailable. The availability can be determined
// by the cluster membership state of the machine that the container is running on. Implement machine membership
// check using Corrossion Admin client.
func filterHealthyContainers(containers []store.ContainerRecord) []store.ContainerRecord {
	healthy := make([]store.ContainerRecord, 0, len(containers))
	for _, cr := range containers {
		if cr.Container.IsHook() {
			continue
		}
		if cr.Container.Healthy() {
			healthy = append(healthy, cr)
		}
	}
	return healthy
}

// generateAndLoadCaddyfile regenerates the Caddyfile from the given containers and loads it into the local Caddy
// if available.
func (c *Controller) generateAndLoadCaddyfile(ctx context.Context, containers []store.ContainerRecord) {
	// Check if Caddy is available before attempting to generate and load config.
	caddyAvailable := c.client.IsAvailable()

	// Skip regeneration when Caddy is available and the containers since the last successful load haven't changed.
	// When Caddy is unavailable we still regenerate to keep the Caddyfile on disk updated.
	fingerprint := fingerprintContainers(containers)
	if caddyAvailable && slices.EqualFunc(fingerprint, c.lastFingerprint, containerFingerprint.Equal) {
		c.log.Debug("Caddy configuration is unchanged.", "path", c.caddyfilePath)
		return
	}

	caddyfile, err := c.generator.Generate(ctx, containers, caddyAvailable)
	if err != nil {
		c.log.Error("Failed to generate Caddyfile configuration.", "err", err)
		return
	}

	if !caddyAvailable {
		// Caddy is not running so the generated Caddyfile should not include user-defined configs thus must be valid.
		// It's safe to write the config to disk so that when Caddy is deployed on this machine, it can pick it up.
		if err = c.writeCaddyfileIfChanged(caddyfile); err != nil {
			c.log.Error("Failed to write Caddyfile to disk.", "err", err)
			return
		}
		c.log.Debug("Caddy is not running on this machine, skipping configuration load.", "path", c.caddyfilePath)
		return
	}

	// Caddy is available, try to load the config which may fail if the config is invalid. Generally, a config can
	// pass the adaptation/validation step but still fail to load, for example, if it references resources that are
	// not available.
	if err = c.client.Load(ctx, caddyfile); err != nil {
		c.log.Error("Failed to load new Caddy configuration into local Caddy instance.",
			"err", err, "path", c.caddyfilePath)
		// Mark the cache stale so the next container change retries the load even if the container set is unchanged.
		c.lastFingerprint = nil
		// Don't write invalid config to disk.
		return
	}
	c.lastFingerprint = fingerprint

	// Config loaded successfully, now write it to disk.
	if err = c.writeCaddyfileIfChanged(caddyfile); err != nil {
		c.log.Error("Failed to write Caddyfile to disk after successful load.", "err", err)
		// Config is already loaded in Caddy, so this is not critical. The next regeneration retries the disk write.
		return
	}

	c.log.Info("New Caddy configuration loaded into local Caddy instance.", "path", c.caddyfilePath)
}

// fingerprintContainers returns a fingerprint of containers that the Caddyfile generator depends on.
func fingerprintContainers(containers []store.ContainerRecord) []containerFingerprint {
	fingerprints := make([]containerFingerprint, len(containers))
	for i, cr := range containers {
		// Ignore ports parsing error as not much we can do about it. The generator just logs them and continues.
		ports, _ := cr.Container.ServicePorts()
		fingerprints[i] = containerFingerprint{
			ID:          cr.Container.ID,
			IP:          cr.Container.UncloudNetworkIP(),
			Ports:       ports,
			CaddyConfig: cr.Container.ServiceSpec.CaddyConfig(),
		}
	}
	slices.SortFunc(fingerprints, func(a, b containerFingerprint) int {
		return strings.Compare(a.ID, b.ID)
	})

	return fingerprints
}

// writeCaddyfileIfChanged writes the Caddyfile content to disk with proper permissions only if its body differs
// from the last successfully written content. The first line of the Caddyfile carries a generation timestamp that
// changes on every regeneration, so it's excluded from the comparison to avoid redundant writes.
func (c *Controller) writeCaddyfileIfChanged(caddyfile string) error {
	if caddyfileBody(caddyfile) == caddyfileBody(c.lastCaddyfile) {
		return nil
	}

	if err := os.WriteFile(c.caddyfilePath, []byte(caddyfile), 0o640); err != nil {
		return fmt.Errorf("write Caddyfile to file '%s': %w", c.caddyfilePath, err)
	}
	if err := fs.Chown(c.caddyfilePath, "", CaddyGroup); err != nil {
		return fmt.Errorf("change owner of Caddyfile '%s': %w", c.caddyfilePath, err)
	}
	c.lastCaddyfile = caddyfile

	return nil
}

// caddyfileBody returns the Caddyfile content without its first line, which carries a generation timestamp that
// rotates on every regeneration.
func caddyfileBody(caddyfile string) string {
	if _, after, ok := strings.Cut(caddyfile, "\n"); ok {
		return after
	}
	return caddyfile
}

func (c *Controller) generateJSONConfig(containers []store.ContainerRecord) error {
	serviceContainers := make([]api.ServiceContainer, len(containers))
	for i, cr := range containers {
		serviceContainers[i] = cr.Container
	}

	config, err := GenerateJSONConfig(serviceContainers, c.machineID)
	if err != nil {
		return err
	}

	configBytes, err := json.MarshalIndent(config, "", "    ")
	if err != nil {
		return fmt.Errorf("marshal Caddy configuration: %w", err)
	}
	configPath := filepath.Join(filepath.Dir(c.caddyfilePath), "caddy.json")

	if err = os.WriteFile(configPath, configBytes, 0o640); err != nil {
		return fmt.Errorf("write Caddy configuration to file '%s': %w", configPath, err)
	}
	if err = fs.Chown(configPath, "", CaddyGroup); err != nil {
		return fmt.Errorf("change owner of Caddy configuration file '%s': %w", configPath, err)
	}

	return nil
}