feat(caddy): make cluster storage for Caddy default, remember 'uc caddy deploy' options

2026-11-10 20:54:17 +00:00 · 2026-11-05 16:03:34 +10:00
parent 76b04dd56c
commit 22b1c68798
13 changed files with 1173 additions and 98 deletions
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"regexp"
+	"strings"
 	"time"

 	"github.com/Masterminds/semver"
@@ -11,6 +12,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/psviderski/uncloud/api/pb"
+	"github.com/psviderski/uncloud/internal/version"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client/deploy"
 	"google.golang.org/protobuf/types/known/emptypb"
@@ -18,11 +20,33 @@ import (

 const (
 	CaddyServiceName = "caddy"
-	// CaddyImage is the official Caddy Docker image on Docker Hub: https://hub.docker.com/_/caddy
-	CaddyImage = "caddy"
+	// CaddyImage is the Uncloud Caddy image that is deployed by default. It includes the Uncloud storage module for
+	// sharing TLS certificates across machines: https://github.com/unlabs-dev/caddy-uncloud
+	CaddyImage = "ghcr.io/unlabs-dev/caddy-uncloud"
+	// DefaultCaddyConfig is the default global Caddyfile. It must be compatible with all versions of CaddyImage allowed
+	// by caddyImageConstraint.
+	DefaultCaddyConfig = "{\n\tstorage uncloud\n}"
+	// CaddyStorageFileSystem is the name of the default Caddy storage module that stores TLS assets locally.
+	CaddyStorageFileSystem = "file_system"
+	// CaddyStorageUncloud is the name of the Caddy storage module that stores TLS assets in the cluster.
+	CaddyStorageUncloud = "uncloud"
+	// MinCaddyDaemonVersion is the minimum daemon version required for 'uc caddy deploy'. Older daemons don't store
+	// service labels that mark the service as managed by this command. Distributed locks were introduced in 0.21.0,
+	// which is the minimum required by the cluster storage even on machines that don't run the Caddy service.
+	MinCaddyDaemonVersion = "0.22.0"
+
+	// officialCaddyImage is the official Caddy Docker image on Docker Hub: https://hub.docker.com/_/caddy
+	officialCaddyImage = "docker.io/library/caddy"
 )

-var caddyImageTagRegex = regexp.MustCompile(`^2\.\d+\.\d+$`)
+var (
+	// caddyImageConstraint limits CaddyImage to the major version that is known to be compatible with
+	// DefaultCaddyConfig.
+	caddyImageConstraint = mustNewConstraint("<1.0.0")
+	// caddyImageTagRegex matches the exact version tags of CaddyImage, skipping floating tags and image variants.
+	caddyImageTagRegex = regexp.MustCompile(`^\d+\.\d+\.\d+$`)
+	caddyStorageRegex  = regexp.MustCompile(`(?m)^[ \t]*storage[ \t]+(\S+)`)
+)

 // CaddyClient provides Caddy operations over its parent Client's connection.
 // The parent client owns the connection and must be used to close it.
@@ -64,19 +88,28 @@ func (c *CaddyClient) Config(ctx context.Context, opts CaddyConfigOptions) (api.
 	return config, nil
 }

-// CaddyDeploymentOptions configures a Caddy reverse proxy deployment.
+// CaddyDeploymentOptions configures a Caddy reverse proxy deployment. The zero value deploys the latest version of
+// CaddyImage with DefaultCaddyConfig to all machines.
 type CaddyDeploymentOptions struct {
-	// Image defaults to the latest stable 2.x.x official Caddy image.
-	// Custom images must include curl and start Caddy with /etc/caddy/Caddyfile.
+	// Image defaults to the latest version of CaddyImage. A non-empty value is marked as custom.
+	// Custom images must include curl and start Caddy with /etc/caddy/Caddyfile. They must also include the Uncloud
+	// storage module if the config uses 'storage uncloud'.
 	Image string
-	// Config contains an optional global Caddyfile.
-	Config    string
+	// Config is the global Caddyfile. Nil means DefaultCaddyConfig. Any other value, even empty, is marked as custom.
+	Config    *string
 	Placement api.Placement
 }

-// NewDeployment creates a new deployment for a Caddy reverse proxy service.
-// The service is deployed in global mode to all machines in the cluster. If the image is not provided, the latest
-// version of the official Caddy Docker image is used.
+// ConfigOrDefault returns the global Caddyfile of the deployment.
+func (o CaddyDeploymentOptions) ConfigOrDefault() string {
+	if o.Config != nil {
+		return *o.Config
+	}
+	return DefaultCaddyConfig
+}
+
+// NewDeployment creates a new managed deployment for a Caddy reverse proxy service.
+// The service is deployed in global mode to all machines in the cluster or the machines set in the placement.
 func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOptions) (*deploy.Deployment, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
@@ -90,6 +123,10 @@ func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOpt

 		image = reference.FamiliarString(latest)
 	}
+	config := opts.ConfigOrDefault()
+	if err := validateCaddyImageStorage(image, config); err != nil {
+		return nil, err
+	}

 	spec := api.ServiceSpec{
 		Container: api.ContainerSpec{
@@ -186,17 +223,119 @@ func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOpt
 		},
 	}

-	if opts.Config != "" {
+	if config != "" {
 		spec.Caddy = &api.CaddySpec{
-			Config: opts.Config,
+			Config: config,
 		}
 	}

+	spec.Labels = map[string]string{api.LabelCaddyManaged: ""}
+	if opts.Image != "" {
+		spec.Labels[api.LabelCaddyImageCustom] = ""
+	}
+	if opts.Config != nil {
+		spec.Labels[api.LabelCaddyConfigCustom] = ""
+	}
+
 	return c.client.NewDeployment(spec, nil), nil
 }

-// latestCaddyImage returns the latest image of the official Caddy Docker image on Docker Hub.
-// The latest image is determined by the latest version tag 2.x.x.
+// CheckDaemonVersions returns an error naming the available machines whose daemons are older than
+// MinCaddyDaemonVersion. Machines that are down are skipped because a deployment doesn't touch them.
+func (c *CaddyClient) CheckDaemonVersions(ctx context.Context) error {
+	machines, err := c.client.ListMachines(ctx, &api.MachineFilter{Available: true})
+	if err != nil {
+		return fmt.Errorf("list machines: %w", err)
+	}
+
+	if outdated := outdatedCaddyDaemons(machines); len(outdated) > 0 {
+		return fmt.Errorf("deploying Caddy requires Uncloud daemon %s or newer on all machines, upgrade these "+
+			"machines and try again: %s. See https://github.com/psviderski/uncloud/releases/latest",
+			MinCaddyDaemonVersion, strings.Join(outdated, ", "))
+	}
+	return nil
+}
+
+// outdatedCaddyDaemons returns descriptions of the machines with outdated daemons. Prereleases and development builds
+// aren't considered outdated because their versions don't show whether they're older than a stable release.
+func outdatedCaddyDaemons(machines api.MachineMembersList) []string {
+	minVersion := semver.MustParse(MinCaddyDaemonVersion)
+
+	var outdated []string
+	for _, m := range machines {
+		v, err := semver.NewVersion(m.Machine.DaemonVersion)
+		if err != nil || v.Prerelease() != "" {
+			continue
+		}
+		if v.LessThan(minVersion) {
+			outdated = append(outdated, fmt.Sprintf("%s (%s)", m.Machine.Name, m.Machine.DaemonVersion))
+		}
+	}
+
+	return outdated
+}
+
+// CaddyStorageModule returns the name of the storage module set in the global options of a Caddyfile.
+func CaddyStorageModule(config string) string {
+	if match := caddyStorageRegex.FindStringSubmatch(config); match != nil {
+		return match[1]
+	}
+	return CaddyStorageFileSystem
+}
+
+// IsDefaultCaddyImage returns whether the image is any version of CaddyImage.
+func IsDefaultCaddyImage(image string) bool {
+	return imageName(image) == CaddyImage
+}
+
+// IsNewerCaddyImage returns whether the image is a newer version of CaddyImage than the other image.
+// It returns false if any of the images isn't a version of CaddyImage.
+func IsNewerCaddyImage(image, other string) bool {
+	version, otherVersion := caddyImageVersion(image), caddyImageVersion(other)
+	return version != nil && otherVersion != nil && version.GreaterThan(otherVersion)
+}
+
+// caddyImageVersion returns the version in the image tag if the image is a version of CaddyImage, or nil otherwise.
+func caddyImageVersion(image string) *semver.Version {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil || named.Name() != CaddyImage {
+		return nil
+	}
+	tagged, ok := named.(reference.Tagged)
+	if !ok {
+		return nil
+	}
+	version, err := semver.NewVersion(tagged.Tag())
+	if err != nil {
+		return nil
+	}
+
+	return version
+}
+
+// imageName returns the normalised name of the image without its tag or digest.
+// It returns an empty string if the image reference is invalid.
+func imageName(image string) string {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return ""
+	}
+	return named.Name()
+}
+
+// validateCaddyImageStorage rejects the official Caddy image with a config that uses the Uncloud storage module
+// because it doesn't include this module. Otherwise, a new container could start from the previously saved Caddyfile
+// and pass its health check, but its routes would stop updating as Caddy would reject the new config.
+func validateCaddyImageStorage(image, config string) error {
+	if imageName(image) == officialCaddyImage && CaddyStorageModule(config) == CaddyStorageUncloud {
+		return fmt.Errorf("official Caddy image '%s' doesn't support 'storage uncloud' in the global Caddy config: "+
+			"use an image that includes the Uncloud storage module or a Caddyfile without this option. "+
+			"See %s/concepts/ingress/cluster-storage-for-caddy", image, version.DocsURL)
+	}
+	return nil
+}
+
+// latestCaddyImage returns the latest version of CaddyImage allowed by caddyImageConstraint.
 func latestCaddyImage(ctx context.Context) (reference.NamedTagged, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
@@ -222,7 +361,7 @@ func latestCaddyImage(ctx context.Context) (reference.NamedTagged, error) {
 	return imageWithTag, nil
 }

-// latestCaddyTag selects the newest stable 2.x.x tag, falling back to latest.
+// latestCaddyTag selects the newest tag matching caddyImageTagRegex and caddyImageConstraint, falling back to latest.
 func latestCaddyTag(tags []string) string {
 	latestTag := "latest"
 	var latestVersion *semver.Version
@@ -232,7 +371,7 @@ func latestCaddyTag(tags []string) string {
 		}

 		v, err := semver.NewVersion(t)
-		if err != nil {
+		if err != nil || !caddyImageConstraint.Check(v) {
 			continue
 		}
 		if latestVersion == nil || v.GreaterThan(latestVersion) {
@@ -243,3 +382,12 @@ func latestCaddyTag(tags []string) string {

 	return latestTag
 }
+
+// mustNewConstraint parses a version constraint and panics if it's invalid.
+func mustNewConstraint(constraint string) *semver.Constraints {
+	c, err := semver.NewConstraint(constraint)
+	if err != nil {
+		panic(fmt.Sprintf("invalid version constraint '%s': %v", constraint, err))
+	}
+	return c
+}