feat(caddy): make cluster storage for Caddy default, remember 'uc caddy deploy' options

README.md

@@ -44,8 +44,8 @@ complexity of Kubernetes.
   [Uncloud DNS](https://github.com/psviderski/uncloud-dns) service.
 * **Automatic HTTPS**: Built-in Caddy reverse proxy handles TLS certificate provisioning and renewal using Let's
   Encrypt.
-* **Cluster storage for TLS certificates**: Native cluster storage for Caddy shares certificates and ACME challenge
+* **Cluster storage for TLS certificates**: [Cluster storage](https://github.com/unlabs-dev/caddy-uncloud) for Caddy
-  tokens across machines and coordinates issuance.
+  shares certificates and ACME challenge tokens across machines and coordinates issuance.
 * **Docker-like CLI**: Familiar commands for managing both infrastructure and applications.
 * **Remote management**: Control your entire infrastructure through SSH access to any single machine in the cluster.
 

cmd/uc/caddy/deploy.go

@@ -16,6 +16,7 @@ import (
 	"github.com/psviderski/uncloud/internal/cli"
 	"github.com/psviderski/uncloud/internal/cli/completion"
 	"github.com/psviderski/uncloud/internal/cli/tui"
+	"github.com/psviderski/uncloud/internal/version"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client"
 	"github.com/spf13/cobra"
@@ -25,6 +26,10 @@ type deployOptions struct {
 	caddyfile string
 	image     string
 	machines  []string
+
+	caddyfileChanged bool
+	imageChanged     bool
+	machinesChanged  bool
 }
 
 func NewDeployCommand() *cobra.Command {
@@ -33,21 +38,62 @@ func NewDeployCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "deploy",
 		Short: "Deploy or upgrade Caddy reverse proxy across all machines in the cluster.",
-		Long: "Deploy or upgrade Caddy reverse proxy across all machines in the cluster.\n" +
+		Long: `Deploy or upgrade Caddy reverse proxy across all machines in the cluster.
-			"A rolling update is performed when updating existing containers to minimise disruption.",
+A rolling update is performed when updating existing containers to minimise disruption.
+
+By default, this deploys the latest version of the Uncloud Caddy image with a global config that enables cluster
+storage for TLS certificates. See https://github.com/unlabs-dev/caddy-uncloud for more details.
+
+Values set with --image, --caddyfile, and --machine are kept for later deployments. Pass an empty value to reset
+a flag to its default.
+
+If this command didn't deploy the current caddy service, for example, it was deployed with 'uc deploy'
+from a Compose file or an older version of uc, the deployment replaces its configuration.
+
+Requires Uncloud daemon ` + client.MinCaddyDaemonVersion + ` or newer on all machines.`,
+		Example: `  # Deploy Caddy or upgrade it to the latest version of the Uncloud Caddy image.
+  uc caddy deploy
+
+  # Deploy a custom image and keep it for later deployments.
+  uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3
+
+  # Go back to the latest version of the Uncloud Caddy image.
+  uc caddy deploy --image ""
+
+  # Deploy a custom image build and a custom global Caddy config and keep both for later deployments.
+  uc caddy deploy --image myorg/caddy-uncloud:latest --caddyfile global.Caddyfile
+
+  # Go back to the default global Caddy config but keep the custom image.
+  uc caddy deploy --caddyfile ""
+
+  # Deploy only to specific machines and keep them for later deployments.
+  uc caddy deploy --machine machine1,machine2
+
+  # Go back to deploying to all machines.
+  uc caddy deploy --machine ""`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			uncli := cmd.Context().Value("cli").(*cli.CLI)
+
+			opts.caddyfileChanged = cmd.Flag("caddyfile").Changed
+			opts.imageChanged = cmd.Flag("image").Changed
+			opts.machinesChanged = cmd.Flag("machine").Changed
+
 			return runDeploy(cmd.Context(), uncli, opts)
 		},
 	}
 
 	cmd.Flags().StringVar(&opts.caddyfile, "caddyfile", "",
-		"Path to a custom global Caddy config (Caddyfile) that will be prepended to the auto-generated Caddy config.")
+		"Path to a custom global Caddy config (Caddyfile) that will be prepended to the auto-generated Caddy config.\n"+
+			"An empty value resets it to the default config that enables cluster storage.\n"+
+			"(default is the current custom config or the default config)")
 	cmd.Flags().StringVar(&opts.image, "image", "",
-		"Caddy Docker image to deploy. (default caddy:LATEST_VERSION)")
+		"Caddy Docker image to deploy. It must include the Uncloud storage module if the Caddy config uses it.\n"+
+			"An empty value resets it to the latest version of the Uncloud Caddy image.\n"+
+			"(default is the current custom image or "+client.CaddyImage+":LATEST_VERSION)")
 	cmd.Flags().StringSliceVarP(&opts.machines, "machine", "m", nil,
-		"Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated "+
+		"Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated list.\n"+
-			"list. (default is all machines)")
+			"An empty value resets it to all machines.\n"+
+			"(default is the current machines or all machines)")
 
 	completion.MachinesFlag(cmd)
 
@@ -55,13 +101,13 @@ func NewDeployCommand() *cobra.Command {
 }
 
 func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
-	caddyfile := ""
+	config := ""
 	if opts.caddyfile != "" {
 		data, err := os.ReadFile(opts.caddyfile)
 		if err != nil {
 			return fmt.Errorf("read Caddyfile: %w", err)
 		}
-		caddyfile = strings.TrimSpace(string(data))
+		config = strings.TrimSpace(string(data))
 	}
 
 	clusterClient, err := uncli.ConnectCluster(ctx)
@@ -70,6 +116,10 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 	}
 	defer clusterClient.Close()
 
+	if err = clusterClient.Caddy.CheckDaemonVersions(ctx); err != nil {
+		return err
+	}
+
 	svc, err := clusterClient.InspectService(ctx, client.CaddyServiceName)
 	if err != nil {
 		if !errors.Is(err, api.ErrNotFound) {
@@ -100,26 +150,48 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 		}
 	}
 
-	if opts.image != "" {
+	current := svc.Spec()
-		fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(opts.image, tui.Green))
+	deployOpts := resolveDeploymentOptions(current, opts, config)
+	configChanged := deployOpts.ConfigOrDefault() != current.CaddyConfig()
+
+	if svc.ID != "" && configChanged {
+		printCaddyConfig("current global Caddy config:", current.CaddyConfig())
 	}
 
 	fmt.Println()
 	fmt.Println("Preparing a deployment plan...")
 
-	placement := api.Placement{
+	d, err := clusterClient.Caddy.NewDeployment(ctx, deployOpts)
-		Machines: cli.ExpandCommaSeparatedValues(opts.machines),
-	}
-	d, err := clusterClient.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{
-		Image: opts.image, Config: caddyfile, Placement: placement,
-	})
 	if err != nil {
 		return fmt.Errorf("create caddy deployment: %w", err)
 	}
+	target := d.Spec
 
-	if opts.image == "" {
+	// The Uncloud Caddy image resolves to the latest version allowed by this CLI, which is older than the running image
-		fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(d.Spec.Container.Image,
+	// if a newer CLI deployed it. Continuing would downgrade the image and replace its default config.
-			tui.Green) + tui.Faint.Render(" (latest stable)"))
+	if deployOpts.Image == "" && client.IsNewerCaddyImage(current.Container.Image, target.Container.Image) {
+		return fmt.Errorf("caddy service runs image '%s' which is newer than the latest version '%s' supported by "+
+			"this uc version: upgrade uc or specify the image with --image",
+			current.Container.Image, target.Container.Image)
+	}
+
+	imageNote := " (latest)"
+	if deployOpts.Image != "" {
+		imageNote = " (custom)"
+		if !opts.imageChanged {
+			imageNote = " (custom, use --image \"\" for the latest version)"
+		}
+	}
+	fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(target.Container.Image, tui.Green) +
+		tui.Faint.Render(imageNote))
+
+	if configChanged {
+		if deployOpts.Config != nil {
+			// The user provided this config so there is no need to print its content.
+			fmt.Println(tui.Faint.Render("target global Caddy config: ") + opts.caddyfile)
+		} else {
+			printCaddyConfig("target global Caddy config: (default)", deployOpts.ConfigOrDefault())
+		}
 	}
 
 	plan, err := d.Plan(ctx)
@@ -131,7 +203,7 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 		fmt.Printf("%s service is up to date.\n", client.CaddyServiceName)
 	} else {
 		if svc.ID == "" {
-			if len(opts.machines) > 0 {
+			if len(deployOpts.Placement.Machines) > 0 {
 				fmt.Println("This will run a Caddy container on each selected machine.")
 			} else {
 				fmt.Println("This will run a Caddy container on each machine.")
@@ -163,6 +235,14 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 		fmt.Println(summary)
 		fmt.Println()
 
+		warnings := deploymentWarnings(current, target, opts)
+		for _, warning := range warnings {
+			tui.PrintWarning(warning)
+		}
+		if len(warnings) > 0 {
+			fmt.Println()
+		}
+
 		title := "Proceed with deployment?"
 		// Include the direct connection or context name in the confirmation prompt to avoid accidentally
 		// deploying to the wrong cluster.
@@ -195,6 +275,98 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 	return UpdateDomainRecords(ctx, clusterClient, uncli.ProgressOut())
 }
 
+// resolveDeploymentOptions returns the deployment options for the flags and the current service spec, which is zero if
+// the service doesn't exist. Each flag works on its own: a specified flag overrides the value and an empty one resets
+// it to the default. A flag that isn't specified keeps the explicitly set value.
+func resolveDeploymentOptions(
+	current api.ServiceSpec, opts deployOptions, config string,
+) client.CaddyDeploymentOptions {
+	var resolved client.CaddyDeploymentOptions
+
+	if opts.imageChanged {
+		resolved.Image = opts.image
+	} else if _, custom := current.Labels[api.LabelCaddyImageCustom]; custom {
+		resolved.Image = current.Container.Image
+	}
+
+	if opts.caddyfileChanged {
+		if opts.caddyfile != "" {
+			resolved.Config = new(config)
+		}
+	} else if _, custom := current.Labels[api.LabelCaddyConfigCustom]; custom {
+		resolved.Config = new(current.CaddyConfig())
+	}
+
+	if opts.machinesChanged {
+		resolved.Placement.Machines = cli.ExpandCommaSeparatedValues(opts.machines)
+	} else if _, managed := current.Labels[api.LabelCaddyManaged]; managed {
+		resolved.Placement = current.Placement
+	}
+
+	return resolved
+}
+
+// deploymentWarnings returns the consequences of deploying the target spec over the current one, which is zero if
+// the service doesn't exist.
+func deploymentWarnings(current, target api.ServiceSpec, opts deployOptions) []string {
+	var warnings []string
+	exists := current.Name != ""
+
+	if _, managed := current.Labels[api.LabelCaddyManaged]; exists && !managed {
+		warnings = append(warnings, "'uc caddy deploy' doesn't manage this caddy service yet, so this deployment "+
+			"replaces its configuration, including any custom global Caddy config.\n"+
+			"If you manage it with a Compose file, cancel and deploy with 'uc deploy' instead.")
+	}
+
+	currentStorage := client.CaddyStorageModule(current.CaddyConfig())
+	targetStorage := client.CaddyStorageModule(target.CaddyConfig())
+	if exists && currentStorage != targetStorage {
+		warning := fmt.Sprintf("Caddy certificate storage changes from %s to %s.\n"+
+			"Caddy doesn't automatically migrate certificates, so all of them will be reissued and HTTPS may fail "+
+			"for those domains until that finishes.\n"+
+			"To migrate them first, see %s/concepts/ingress/cluster-storage-for-caddy",
+			caddyStorageName(currentStorage), caddyStorageName(targetStorage), version.DocsURL)
+		if !opts.caddyfileChanged {
+			warning += "\nTo keep the current storage, specify a custom Caddyfile that uses it with --caddyfile."
+		}
+		warnings = append(warnings, warning)
+	}
+
+	imageOrConfigChanged := current.Container.Image != target.Container.Image ||
+		current.CaddyConfig() != target.CaddyConfig()
+	if targetStorage == client.CaddyStorageUncloud && imageOrConfigChanged &&
+		!client.IsDefaultCaddyImage(target.Container.Image) {
+		warnings = append(warnings, fmt.Sprintf("Image '%s' must include the Uncloud storage module to use "+
+			"'storage uncloud'. Otherwise, Caddy won't start or apply the new config.\n"+
+			"Check 'uc caddy config' after the deployment for errors.", target.Container.Image))
+	}
+
+	return warnings
+}
+
+// caddyStorageName returns a user-friendly name of the Caddy storage module.
+func caddyStorageName(module string) string {
+	switch module {
+	case client.CaddyStorageFileSystem:
+		return "local"
+	case client.CaddyStorageUncloud:
+		return "cluster"
+	default:
+		return "'" + module + "'"
+	}
+}
+
+// printCaddyConfig prints the title and the indented config or "none" if it's empty.
+func printCaddyConfig(title, config string) {
+	if config == "" {
+		lipgloss.Println(tui.Faint.Render(title) + " none")
+		return
+	}
+
+	lipgloss.Println(tui.Faint.Render(title))
+	fmt.Println(config)
+}
+
 func UpdateDomainRecords(ctx context.Context, clusterClient *client.Client, progressOut *streams.Out) error {
 	domain, err := clusterClient.GetDomain(ctx)
 	if err != nil {

cmd/uc/caddy/deploy_test.go

@@ -0,0 +1,263 @@
+package caddy
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/psviderski/uncloud/pkg/client"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestResolveDeploymentOptions(t *testing.T) {
+	t.Parallel()
+
+	const (
+		customImage  = "example.com/caddy:1.2.3"
+		customConfig = "{\n\tdebug\n}"
+	)
+	machines := api.Placement{Machines: []string{"m1", "m2"}}
+
+	// A service with a custom image, config, and machines that has the given labels.
+	service := func(labels ...string) api.ServiceSpec {
+		spec := api.ServiceSpec{
+			Name:      client.CaddyServiceName,
+			Caddy:     &api.CaddySpec{Config: customConfig},
+			Container: api.ContainerSpec{Image: customImage},
+			Placement: machines,
+		}
+		if len(labels) > 0 {
+			spec.Labels = make(map[string]string, len(labels))
+			for _, label := range labels {
+				spec.Labels[label] = ""
+			}
+		}
+		return spec
+	}
+
+	tests := []struct {
+		name    string
+		current api.ServiceSpec
+		opts    deployOptions
+		config  string
+		want    client.CaddyDeploymentOptions
+	}{
+		{
+			name: "no service gets the defaults",
+		},
+		{
+			name:    "service that isn't managed gets the defaults",
+			current: service(),
+		},
+		{
+			name:    "managed service follows the default image and config and keeps machines",
+			current: service(api.LabelCaddyManaged),
+			want:    client.CaddyDeploymentOptions{Placement: machines},
+		},
+		{
+			name:    "custom image is kept and config follows the default",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom),
+			want:    client.CaddyDeploymentOptions{Image: customImage, Placement: machines},
+		},
+		{
+			name:    "custom config is kept and image follows the latest version",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyConfigCustom),
+			want:    client.CaddyDeploymentOptions{Config: new(customConfig), Placement: machines},
+		},
+		{
+			name:    "custom image and config are kept",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom, api.LabelCaddyConfigCustom),
+			want:    client.CaddyDeploymentOptions{Image: customImage, Config: new(customConfig), Placement: machines},
+		},
+		{
+			name: "custom empty config is kept",
+			current: api.ServiceSpec{
+				Name:   client.CaddyServiceName,
+				Labels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
+			},
+			want: client.CaddyDeploymentOptions{Config: new("")},
+		},
+		{
+			name:    "image flag overrides the current image",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom),
+			opts:    deployOptions{image: "caddy:2", imageChanged: true},
+			want:    client.CaddyDeploymentOptions{Image: "caddy:2", Placement: machines},
+		},
+		{
+			name:    "empty image flag resets to the latest version",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom),
+			opts:    deployOptions{imageChanged: true},
+			want:    client.CaddyDeploymentOptions{Placement: machines},
+		},
+		{
+			name:    "caddyfile flag overrides the current config",
+			current: service(api.LabelCaddyManaged),
+			opts:    deployOptions{caddyfile: "global.Caddyfile", caddyfileChanged: true},
+			config:  "{\n\tstorage redis\n}",
+			want:    client.CaddyDeploymentOptions{Config: new("{\n\tstorage redis\n}"), Placement: machines},
+		},
+		{
+			name:    "caddyfile flag with an empty file sets a custom empty config",
+			current: service(api.LabelCaddyManaged),
+			opts:    deployOptions{caddyfile: "empty.Caddyfile", caddyfileChanged: true},
+			want:    client.CaddyDeploymentOptions{Config: new(""), Placement: machines},
+		},
+		{
+			name:    "empty caddyfile flag resets to the default config",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyConfigCustom),
+			opts:    deployOptions{caddyfileChanged: true},
+			want:    client.CaddyDeploymentOptions{Placement: machines},
+		},
+		{
+			name:    "machine flag overrides the current machines",
+			current: service(api.LabelCaddyManaged),
+			opts:    deployOptions{machines: []string{"m3,m4", "m5"}, machinesChanged: true},
+			want:    client.CaddyDeploymentOptions{Placement: api.Placement{Machines: []string{"m3", "m4", "m5"}}},
+		},
+		{
+			name:    "empty machine flag resets to all machines",
+			current: service(api.LabelCaddyManaged),
+			opts:    deployOptions{machines: []string{}, machinesChanged: true},
+		},
+		{
+			name:    "flags apply to a service that isn't managed",
+			current: service(),
+			opts: deployOptions{
+				image: "caddy:2", imageChanged: true,
+				caddyfile: "global.Caddyfile", caddyfileChanged: true,
+				machines: []string{"m3"}, machinesChanged: true,
+			},
+			config: customConfig,
+			want: client.CaddyDeploymentOptions{
+				Image:     "caddy:2",
+				Config:    new(customConfig),
+				Placement: api.Placement{Machines: []string{"m3"}},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, resolveDeploymentOptions(tt.current, tt.opts, tt.config))
+		})
+	}
+}
+
+func TestDeploymentWarnings(t *testing.T) {
+	t.Parallel()
+
+	const (
+		defaultImage = client.CaddyImage + ":0.1.3"
+		customImage  = "example.com/caddy:1.2.3"
+		localConfig  = "{\n\tdebug\n}"
+		override     = "doesn't manage this caddy service yet"
+		storage      = "certificate storage changes from"
+		keepStorage  = "To keep the current storage"
+		module       = "must include the Uncloud storage module"
+	)
+	managed := map[string]string{api.LabelCaddyManaged: ""}
+
+	service := func(image, config string, labels map[string]string) api.ServiceSpec {
+		spec := api.ServiceSpec{
+			Name:      client.CaddyServiceName,
+			Container: api.ContainerSpec{Image: image},
+			Labels:    labels,
+		}
+		if config != "" {
+			spec.Caddy = &api.CaddySpec{Config: config}
+		}
+		return spec
+	}
+
+	tests := []struct {
+		name    string
+		current api.ServiceSpec
+		target  api.ServiceSpec
+		opts    deployOptions
+		want    []string
+		notWant []string
+	}{
+		{
+			name:    "new service with defaults has no warnings",
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			notWant: []string{override, storage, module},
+		},
+		{
+			name:    "new service with a custom image",
+			target:  service(customImage, client.DefaultCaddyConfig, managed),
+			want:    []string{module},
+			notWant: []string{override, storage},
+		},
+		{
+			name:    "service that isn't managed moves to cluster storage",
+			current: service("caddy:2.11.4", "", nil),
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			want:    []string{override, "from local to cluster", keepStorage},
+			notWant: []string{module},
+		},
+		{
+			name:    "service that isn't managed already uses cluster storage",
+			current: service(defaultImage, client.DefaultCaddyConfig, nil),
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			want:    []string{override},
+			notWant: []string{storage, module},
+		},
+		{
+			name:    "managed service without changes has no warnings",
+			current: service(defaultImage, client.DefaultCaddyConfig, managed),
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			notWant: []string{override, storage, module},
+		},
+		{
+			name:    "managed service upgrades the default image",
+			current: service(client.CaddyImage+":0.1.2", client.DefaultCaddyConfig, managed),
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			notWant: []string{override, storage, module},
+		},
+		{
+			name:    "custom caddyfile moves to local storage",
+			current: service(defaultImage, client.DefaultCaddyConfig, managed),
+			target:  service(defaultImage, localConfig, managed),
+			opts:    deployOptions{caddyfile: "global.Caddyfile", caddyfileChanged: true},
+			want:    []string{"from cluster to local"},
+			notWant: []string{override, keepStorage, module},
+		},
+		{
+			name:    "other storage module is named",
+			current: service(defaultImage, "{\n\tstorage redis\n}", managed),
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			want:    []string{"from 'redis' to cluster", keepStorage},
+		},
+		{
+			name:    "custom image is unchanged with the same config",
+			current: service(customImage, client.DefaultCaddyConfig, managed),
+			target:  service(customImage, client.DefaultCaddyConfig, managed),
+			notWant: []string{override, storage, module},
+		},
+		{
+			name:    "custom image receives a changed default config",
+			current: service(customImage, "{\n\tstorage uncloud\n\tdebug\n}", managed),
+			target:  service(customImage, client.DefaultCaddyConfig, managed),
+			want:    []string{module},
+			notWant: []string{override, storage},
+		},
+		{
+			name:    "custom image with local storage",
+			current: service(defaultImage, localConfig, managed),
+			target:  service(customImage, localConfig, managed),
+			notWant: []string{override, storage, module},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			warnings := strings.Join(deploymentWarnings(tt.current, tt.target, tt.opts), "\n")
+			for _, want := range tt.want {
+				assert.Contains(t, warnings, want)
+			}
+			for _, notWant := range tt.notWant {
+				assert.NotContains(t, warnings, notWant)
+			}
+		})
+	}
+}

cmd/uc/machine/add.go

@@ -187,14 +187,12 @@ func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine,
 	}
 	fmt.Println("Machine joined the cluster.")
 
-	// TODO: scale the existing Caddy service to the new machine instead of running a new deployment
+	// Deploy the current caddy service spec to run a container on the added machine. This keeps the service
-	//  that may cause a small downtime.
+	// configuration however it was deployed and shouldn't change the containers on other machines if specs on them
-	// Deploy a Caddy service container to the added machine. If caddy service is already deployed on other machines,
+	// haven't diverged.
-	// use the deployed image version.
 	// NOTE: We use the cluster client to inspect and scale the Caddy service because the newly added machine may have
 	// issues accessing the API of existing machines in the cluster.
 	// See the issue for more details: https://github.com/psviderski/uncloud/issues/65.
-	caddyImage := ""
 	caddySvc, err := clusterClient.InspectService(ctx, client.CaddyServiceName)
 	if err != nil {
 		if errors.Is(err, api.ErrNotFound) {
@@ -203,26 +201,10 @@ func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine,
 		}
 		return fmt.Errorf("inspect caddy service: %w", err)
 	}
-	caddyImage = caddySvc.Containers[0].Container.Config.Image
-	// Find the latest created container and use its image.
-	var latestCreated time.Time
-	for _, c := range caddySvc.Containers[1:] {
-		created, err := time.Parse(time.RFC3339Nano, c.Container.Created)
-		if err != nil {
-			continue
-		}
-		if created.After(latestCreated) {
-			latestCreated = created
-			caddyImage = c.Container.Config.Image
-		}
-	}
 
 	fmt.Println()
 	fmt.Println("Preparing Caddy deployment...")
-	d, err := clusterClient.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: caddyImage})
+	d := clusterClient.NewDeployment(caddySvc.Spec(), nil)
-	if err != nil {
-		return fmt.Errorf("create caddy deployment: %w", err)
-	}
 
 	plan, err := d.Plan(ctx)
 	if err != nil {

pkg/api/caddy.go

@@ -5,6 +5,15 @@ import (
 	"time"
 )
 
+const (
+	// LabelCaddyManaged marks the caddy service as managed by 'uc caddy deploy'.
+	LabelCaddyManaged = LabelReservedPrefix + "caddy.managed"
+	// LabelCaddyImageCustom marks the caddy image as explicitly set instead of following the latest default image.
+	LabelCaddyImageCustom = LabelReservedPrefix + "caddy.image.custom"
+	// LabelCaddyConfigCustom marks the global Caddy config as explicitly set instead of following the default config.
+	LabelCaddyConfigCustom = LabelReservedPrefix + "caddy.config.custom"
+)
+
 // CaddyConfig is the saved Caddy configuration on a machine.
 type CaddyConfig struct {
 	Caddyfile string

pkg/client/caddy.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/Masterminds/semver"
@@ -11,6 +12,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/psviderski/uncloud/api/pb"
+	"github.com/psviderski/uncloud/internal/version"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client/deploy"
 	"google.golang.org/protobuf/types/known/emptypb"
@@ -18,11 +20,33 @@ import (
 
 const (
 	CaddyServiceName = "caddy"
-	// CaddyImage is the official Caddy Docker image on Docker Hub: https://hub.docker.com/_/caddy
+	// CaddyImage is the Uncloud Caddy image that is deployed by default. It includes the Uncloud storage module for
-	CaddyImage = "caddy"
+	// sharing TLS certificates across machines: https://github.com/unlabs-dev/caddy-uncloud
+	CaddyImage = "ghcr.io/unlabs-dev/caddy-uncloud"
+	// DefaultCaddyConfig is the default global Caddyfile. It must be compatible with all versions of CaddyImage allowed
+	// by caddyImageConstraint.
+	DefaultCaddyConfig = "{\n\tstorage uncloud\n}"
+	// CaddyStorageFileSystem is the name of the default Caddy storage module that stores TLS assets locally.
+	CaddyStorageFileSystem = "file_system"
+	// CaddyStorageUncloud is the name of the Caddy storage module that stores TLS assets in the cluster.
+	CaddyStorageUncloud = "uncloud"
+	// MinCaddyDaemonVersion is the minimum daemon version required for 'uc caddy deploy'. Older daemons don't store
+	// service labels that mark the service as managed by this command. Distributed locks were introduced in 0.21.0,
+	// which is the minimum required by the cluster storage even on machines that don't run the Caddy service.
+	MinCaddyDaemonVersion = "0.22.0"
+
+	// officialCaddyImage is the official Caddy Docker image on Docker Hub: https://hub.docker.com/_/caddy
+	officialCaddyImage = "docker.io/library/caddy"
 )
 
-var caddyImageTagRegex = regexp.MustCompile(`^2\.\d+\.\d+$`)
+var (
+	// caddyImageConstraint limits CaddyImage to the major version that is known to be compatible with
+	// DefaultCaddyConfig.
+	caddyImageConstraint = mustNewConstraint("<1.0.0")
+	// caddyImageTagRegex matches the exact version tags of CaddyImage, skipping floating tags and image variants.
+	caddyImageTagRegex = regexp.MustCompile(`^\d+\.\d+\.\d+$`)
+	caddyStorageRegex  = regexp.MustCompile(`(?m)^[ \t]*storage[ \t]+(\S+)`)
+)
 
 // CaddyClient provides Caddy operations over its parent Client's connection.
 // The parent client owns the connection and must be used to close it.
@@ -64,19 +88,28 @@ func (c *CaddyClient) Config(ctx context.Context, opts CaddyConfigOptions) (api.
 	return config, nil
 }
 
-// CaddyDeploymentOptions configures a Caddy reverse proxy deployment.
+// CaddyDeploymentOptions configures a Caddy reverse proxy deployment. The zero value deploys the latest version of
+// CaddyImage with DefaultCaddyConfig to all machines.
 type CaddyDeploymentOptions struct {
-	// Image defaults to the latest stable 2.x.x official Caddy image.
+	// Image defaults to the latest version of CaddyImage. A non-empty value is marked as custom.
-	// Custom images must include curl and start Caddy with /etc/caddy/Caddyfile.
+	// Custom images must include curl and start Caddy with /etc/caddy/Caddyfile. They must also include the Uncloud
+	// storage module if the config uses 'storage uncloud'.
 	Image string
-	// Config contains an optional global Caddyfile.
+	// Config is the global Caddyfile. Nil means DefaultCaddyConfig. Any other value, even empty, is marked as custom.
-	Config    string
+	Config    *string
 	Placement api.Placement
 }
 
-// NewDeployment creates a new deployment for a Caddy reverse proxy service.
+// ConfigOrDefault returns the global Caddyfile of the deployment.
-// The service is deployed in global mode to all machines in the cluster. If the image is not provided, the latest
+func (o CaddyDeploymentOptions) ConfigOrDefault() string {
-// version of the official Caddy Docker image is used.
+	if o.Config != nil {
+		return *o.Config
+	}
+	return DefaultCaddyConfig
+}
+
+// NewDeployment creates a new managed deployment for a Caddy reverse proxy service.
+// The service is deployed in global mode to all machines in the cluster or the machines set in the placement.
 func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOptions) (*deploy.Deployment, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
@@ -90,6 +123,10 @@ func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOpt
 
 		image = reference.FamiliarString(latest)
 	}
+	config := opts.ConfigOrDefault()
+	if err := validateCaddyImageStorage(image, config); err != nil {
+		return nil, err
+	}
 
 	spec := api.ServiceSpec{
 		Container: api.ContainerSpec{
@@ -186,17 +223,119 @@ func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOpt
 		},
 	}
 
-	if opts.Config != "" {
+	if config != "" {
 		spec.Caddy = &api.CaddySpec{
-			Config: opts.Config,
+			Config: config,
 		}
 	}
 
+	spec.Labels = map[string]string{api.LabelCaddyManaged: ""}
+	if opts.Image != "" {
+		spec.Labels[api.LabelCaddyImageCustom] = ""
+	}
+	if opts.Config != nil {
+		spec.Labels[api.LabelCaddyConfigCustom] = ""
+	}
+
 	return c.client.NewDeployment(spec, nil), nil
 }
 
-// latestCaddyImage returns the latest image of the official Caddy Docker image on Docker Hub.
+// CheckDaemonVersions returns an error naming the available machines whose daemons are older than
-// The latest image is determined by the latest version tag 2.x.x.
+// MinCaddyDaemonVersion. Machines that are down are skipped because a deployment doesn't touch them.
+func (c *CaddyClient) CheckDaemonVersions(ctx context.Context) error {
+	machines, err := c.client.ListMachines(ctx, &api.MachineFilter{Available: true})
+	if err != nil {
+		return fmt.Errorf("list machines: %w", err)
+	}
+
+	if outdated := outdatedCaddyDaemons(machines); len(outdated) > 0 {
+		return fmt.Errorf("deploying Caddy requires Uncloud daemon %s or newer on all machines, upgrade these "+
+			"machines and try again: %s. See https://github.com/psviderski/uncloud/releases/latest",
+			MinCaddyDaemonVersion, strings.Join(outdated, ", "))
+	}
+	return nil
+}
+
+// outdatedCaddyDaemons returns descriptions of the machines with outdated daemons. Prereleases and development builds
+// aren't considered outdated because their versions don't show whether they're older than a stable release.
+func outdatedCaddyDaemons(machines api.MachineMembersList) []string {
+	minVersion := semver.MustParse(MinCaddyDaemonVersion)
+
+	var outdated []string
+	for _, m := range machines {
+		v, err := semver.NewVersion(m.Machine.DaemonVersion)
+		if err != nil || v.Prerelease() != "" {
+			continue
+		}
+		if v.LessThan(minVersion) {
+			outdated = append(outdated, fmt.Sprintf("%s (%s)", m.Machine.Name, m.Machine.DaemonVersion))
+		}
+	}
+
+	return outdated
+}
+
+// CaddyStorageModule returns the name of the storage module set in the global options of a Caddyfile.
+func CaddyStorageModule(config string) string {
+	if match := caddyStorageRegex.FindStringSubmatch(config); match != nil {
+		return match[1]
+	}
+	return CaddyStorageFileSystem
+}
+
+// IsDefaultCaddyImage returns whether the image is any version of CaddyImage.
+func IsDefaultCaddyImage(image string) bool {
+	return imageName(image) == CaddyImage
+}
+
+// IsNewerCaddyImage returns whether the image is a newer version of CaddyImage than the other image.
+// It returns false if any of the images isn't a version of CaddyImage.
+func IsNewerCaddyImage(image, other string) bool {
+	version, otherVersion := caddyImageVersion(image), caddyImageVersion(other)
+	return version != nil && otherVersion != nil && version.GreaterThan(otherVersion)
+}
+
+// caddyImageVersion returns the version in the image tag if the image is a version of CaddyImage, or nil otherwise.
+func caddyImageVersion(image string) *semver.Version {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil || named.Name() != CaddyImage {
+		return nil
+	}
+	tagged, ok := named.(reference.Tagged)
+	if !ok {
+		return nil
+	}
+	version, err := semver.NewVersion(tagged.Tag())
+	if err != nil {
+		return nil
+	}
+
+	return version
+}
+
+// imageName returns the normalised name of the image without its tag or digest.
+// It returns an empty string if the image reference is invalid.
+func imageName(image string) string {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return ""
+	}
+	return named.Name()
+}
+
+// validateCaddyImageStorage rejects the official Caddy image with a config that uses the Uncloud storage module
+// because it doesn't include this module. Otherwise, a new container could start from the previously saved Caddyfile
+// and pass its health check, but its routes would stop updating as Caddy would reject the new config.
+func validateCaddyImageStorage(image, config string) error {
+	if imageName(image) == officialCaddyImage && CaddyStorageModule(config) == CaddyStorageUncloud {
+		return fmt.Errorf("official Caddy image '%s' doesn't support 'storage uncloud' in the global Caddy config: "+
+			"use an image that includes the Uncloud storage module or a Caddyfile without this option. "+
+			"See %s/concepts/ingress/cluster-storage-for-caddy", image, version.DocsURL)
+	}
+	return nil
+}
+
+// latestCaddyImage returns the latest version of CaddyImage allowed by caddyImageConstraint.
 func latestCaddyImage(ctx context.Context) (reference.NamedTagged, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
@@ -222,7 +361,7 @@ func latestCaddyImage(ctx context.Context) (reference.NamedTagged, error) {
 	return imageWithTag, nil
 }
 
-// latestCaddyTag selects the newest stable 2.x.x tag, falling back to latest.
+// latestCaddyTag selects the newest tag matching caddyImageTagRegex and caddyImageConstraint, falling back to latest.
 func latestCaddyTag(tags []string) string {
 	latestTag := "latest"
 	var latestVersion *semver.Version
@@ -232,7 +371,7 @@ func latestCaddyTag(tags []string) string {
 		}
 
 		v, err := semver.NewVersion(t)
-		if err != nil {
+		if err != nil || !caddyImageConstraint.Check(v) {
 			continue
 		}
 		if latestVersion == nil || v.GreaterThan(latestVersion) {
@@ -243,3 +382,12 @@ func latestCaddyTag(tags []string) string {
 
 	return latestTag
 }
+
+// mustNewConstraint parses a version constraint and panics if it's invalid.
+func mustNewConstraint(constraint string) *semver.Constraints {
+	c, err := semver.NewConstraint(constraint)
+	if err != nil {
+		panic(fmt.Sprintf("invalid version constraint '%s': %v", constraint, err))
+	}
+	return c
+}

pkg/client/caddy_test.go

@@ -5,6 +5,8 @@ import (
 	"testing"
 
 	"github.com/distribution/reference"
+	"github.com/psviderski/uncloud/api/pb"
+	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -15,7 +17,7 @@ func TestLatestCaddyImage(t *testing.T) {
 	image, err := latestCaddyImage(context.Background())
 	require.NoError(t, err)
 
-	assert.Regexp(t, `^caddy:2\.\d+\.\d+$`, reference.FamiliarString(image))
+	assert.Regexp(t, `^ghcr\.io/unlabs-dev/caddy-uncloud:0\.\d+\.\d+$`, reference.FamiliarString(image))
 }
 
 func TestLatestCaddyTag(t *testing.T) {
@@ -27,14 +29,209 @@ func TestLatestCaddyTag(t *testing.T) {
 		want string
 	}{
 		{name: "empty", want: "latest"},
-		{name: "semantic ordering", tags: []string{"2.9.9", "2.11.4", "2.10.0", "2.11.3"}, want: "2.11.4"},
+		{name: "semantic ordering", tags: []string{"0.1.9", "0.1.11", "0.1.10", "0.1.2"}, want: "0.1.11"},
-		{name: "ignore other versions and variants", tags: []string{
+		{name: "newest minor version", tags: []string{"0.1.3", "0.2.0", "0.0.9"}, want: "0.2.0"},
-			"latest", "1.0.0", "3.0.0", "2.12.0-rc.1", "2.12.0-alpine", "2.12", "v2.12.0", "2.11.4",
+		{name: "ignore newer major versions", tags: []string{"1.0.0", "0.1.3", "2.3.4", "1.0.1"}, want: "0.1.3"},
-		}, want: "2.11.4"},
+		{name: "ignore variants", tags: []string{
-		{name: "no stable version", tags: []string{"builder", "2.12.0-beta.1"}, want: "latest"},
+			"latest", "0.1.4-rc.1", "0.1.4-alpine", "0.1", "v0.1.4", "1.0.0-rc.1", "0.1.3",
+		}, want: "0.1.3"},
+		{name: "no stable version", tags: []string{"builder", "0.1.4-beta.1"}, want: "latest"},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			assert.Equal(t, tt.want, latestCaddyTag(tt.tags))
 		})
 	}
 }
+
+func TestDefaultCaddyConfig_UsesUncloudStorage(t *testing.T) {
+	t.Parallel()
+
+	assert.Equal(t, CaddyStorageUncloud, CaddyStorageModule(DefaultCaddyConfig))
+}
+
+func TestCaddyStorageModule(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		name   string
+		config string
+		want   string
+	}{
+		{name: "empty config", want: CaddyStorageFileSystem},
+		{name: "no storage option", config: "{\n\tdebug\n}", want: CaddyStorageFileSystem},
+		{name: "uncloud", config: "{\n\tstorage uncloud\n}", want: CaddyStorageUncloud},
+		{name: "uncloud with options", config: "{\n\tstorage uncloud {\n\t\tfoo bar\n\t}\n}", want: CaddyStorageUncloud},
+		{name: "spaces around", config: "{\n    storage   uncloud  \n}", want: CaddyStorageUncloud},
+		{name: "other module", config: "{\n\tstorage redis\n}", want: "redis"},
+		{name: "explicit file system", config: "{\n\tstorage file_system /data\n}", want: CaddyStorageFileSystem},
+		{name: "commented out", config: "{\n\t# storage uncloud\n}", want: CaddyStorageFileSystem},
+		{name: "similar option", config: "{\n\tstorage_clean_interval 24h\n}", want: CaddyStorageFileSystem},
+		{name: "no module on the same line", config: "{\n\tstorage\n\tdebug\n}", want: CaddyStorageFileSystem},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, CaddyStorageModule(tt.config))
+		})
+	}
+}
+
+func TestIsDefaultCaddyImage(t *testing.T) {
+	t.Parallel()
+
+	assert.True(t, IsDefaultCaddyImage("ghcr.io/unlabs-dev/caddy-uncloud"))
+	assert.True(t, IsDefaultCaddyImage("ghcr.io/unlabs-dev/caddy-uncloud:0.1.3"))
+	assert.False(t, IsDefaultCaddyImage("caddy:2.11.4"))
+	assert.False(t, IsDefaultCaddyImage("example.com/unlabs-dev/caddy-uncloud:0.1.3"))
+	assert.False(t, IsDefaultCaddyImage(""))
+}
+
+func TestIsNewerCaddyImage(t *testing.T) {
+	t.Parallel()
+
+	const repo = "ghcr.io/unlabs-dev/caddy-uncloud"
+	for _, tt := range []struct {
+		name  string
+		image string
+		other string
+		want  bool
+	}{
+		{name: "newer major version", image: repo + ":1.0.0", other: repo + ":0.1.3", want: true},
+		{name: "newer minor version", image: repo + ":0.2.0", other: repo + ":0.1.3", want: true},
+		{name: "same", image: repo + ":0.1.3", other: repo + ":0.1.3"},
+		{name: "older", image: repo + ":0.1.2", other: repo + ":0.1.3"},
+		{name: "different repo", image: "example.com/custom:1.0.0", other: repo + ":0.1.3"},
+		{name: "official image", image: "caddy:2.11.4", other: repo + ":0.1.3"},
+		{name: "tag without version", image: repo + ":latest", other: repo + ":0.1.3"},
+		{name: "no tag", image: repo, other: repo + ":0.1.3"},
+		{name: "fallback to latest tag", image: repo + ":1.0.0", other: repo + ":latest"},
+		{name: "no current image", other: repo + ":0.1.3"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, IsNewerCaddyImage(tt.image, tt.other))
+		})
+	}
+}
+
+func TestCaddyClientNewDeployment(t *testing.T) {
+	t.Parallel()
+
+	const customImage = "example.com/caddy:1.2.3"
+	for _, tt := range []struct {
+		name       string
+		opts       CaddyDeploymentOptions
+		wantConfig string
+		wantLabels map[string]string
+	}{
+		{
+			name:       "defaults",
+			wantConfig: DefaultCaddyConfig,
+			wantLabels: map[string]string{api.LabelCaddyManaged: ""},
+		},
+		{
+			name:       "custom image",
+			opts:       CaddyDeploymentOptions{Image: customImage},
+			wantConfig: DefaultCaddyConfig,
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyImageCustom: ""},
+		},
+		{
+			name:       "custom config",
+			opts:       CaddyDeploymentOptions{Config: new("{\n\tdebug\n}")},
+			wantConfig: "{\n\tdebug\n}",
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
+		},
+		{
+			name:       "custom empty config",
+			opts:       CaddyDeploymentOptions{Config: new("")},
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
+		},
+		{
+			name:       "custom config matching the default is still custom",
+			opts:       CaddyDeploymentOptions{Config: new(DefaultCaddyConfig)},
+			wantConfig: DefaultCaddyConfig,
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
+		},
+		{
+			name: "custom image and config",
+			opts: CaddyDeploymentOptions{
+				Image:  customImage,
+				Config: new("{\n\tdebug\n}"),
+			},
+			wantConfig: "{\n\tdebug\n}",
+			wantLabels: map[string]string{
+				api.LabelCaddyManaged:      "",
+				api.LabelCaddyImageCustom:  "",
+				api.LabelCaddyConfigCustom: "",
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &CaddyClient{client: &Client{}}
+			d, err := c.NewDeployment(context.Background(), tt.opts)
+			require.NoError(t, err)
+
+			assert.Equal(t, tt.wantConfig, d.Spec.CaddyConfig())
+			assert.Equal(t, tt.wantLabels, d.Spec.Labels)
+		})
+	}
+
+	t.Run("official image needs a config without cluster storage", func(t *testing.T) {
+		c := &CaddyClient{client: &Client{}}
+		_, err := c.NewDeployment(context.Background(), CaddyDeploymentOptions{Image: "caddy:2.11.4"})
+		require.ErrorContains(t, err, "doesn't support 'storage uncloud'")
+
+		_, err = c.NewDeployment(context.Background(), CaddyDeploymentOptions{Image: "caddy:2.11.4", Config: new("")})
+		require.NoError(t, err)
+	})
+}
+
+func TestValidateCaddyImageStorage(t *testing.T) {
+	t.Parallel()
+
+	const storageConfig = "{\n\tstorage uncloud\n}"
+	for _, tt := range []struct {
+		name    string
+		image   string
+		config  string
+		wantErr bool
+	}{
+		{name: "official image with uncloud storage", image: "caddy:2.11.4", config: storageConfig, wantErr: true},
+		{name: "official image with registry", image: "docker.io/library/caddy:2", config: storageConfig, wantErr: true},
+		{name: "official image without config", image: "caddy:2.11.4"},
+		{name: "official image with other config", image: "caddy:2.11.4", config: "{\n\tdebug\n}"},
+		{name: "default image with uncloud storage", image: CaddyImage + ":0.1.3", config: storageConfig},
+		{name: "custom image with uncloud storage", image: "example.com/caddy:1", config: storageConfig},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateCaddyImageStorage(tt.image, tt.config)
+			if tt.wantErr {
+				require.ErrorContains(t, err, "doesn't support 'storage uncloud'")
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestOutdatedCaddyDaemons(t *testing.T) {
+	t.Parallel()
+
+	machine := func(name, version string) *pb.MachineMember {
+		return &pb.MachineMember{Machine: &pb.MachineInfo{Name: name, DaemonVersion: version}}
+	}
+
+	assert.Equal(t, []string{"old (0.21.0)", "older (0.20.0)", "unsupported (0.9.3)"},
+		outdatedCaddyDaemons(api.MachineMembersList{
+			machine("old", "0.21.0"),
+			machine("older", "0.20.0"),
+			machine("unsupported", "0.9.3"),
+			machine("no-version", ""),
+			machine("required", "0.22.0"),
+			machine("newer", "0.23.1"),
+			machine("major", "1.0.0"),
+			machine("nightly", "0.22.0-nightly-abc1234"),
+			machine("old-dev-build", "v0.21.0-5-g6c7ddeb-dirty"),
+			machine("local-build", "999.0.0-dev"),
+			machine("not-semver", "6c7ddeb"),
+		}))
+	assert.Empty(t, outdatedCaddyDaemons(api.MachineMembersList{machine("required", "0.22.0")}))
+	assert.Empty(t, outdatedCaddyDaemons(nil))
+}

pkg/client/deploy/deploy.go

@@ -134,6 +134,22 @@ func (sp *ServicePlan) Format() string {
 		specTable.Row(mod, "image:", formatImageDiff(oldSpec.Container.Image, sp.Spec.Container.Image))
 	}
 
+	// An existing service without an old spec only has new containers added to up-to-date ones so its Caddy config and
+	// labels didn't change.
+	specChanged := sp.IsNewService || oldSpec != nil
+	var old api.ServiceSpec
+	if oldSpec != nil {
+		old = *oldSpec
+	}
+
+	// Caddy config row.
+	if specChanged {
+		mod, value := formatCaddyConfigDiff(old.CaddyConfig(), sp.Spec.CaddyConfig(), sp.IsNewService)
+		if value != "" {
+			specTable.Row(mod, "caddy config:", value)
+		}
+	}
+
 	// Replicas row for replicated services.
 	if sp.Spec.Mode == api.ServiceModeReplicated {
 		replicasStr := fmt.Sprintf("%d", sp.Spec.Replicas)
@@ -153,15 +169,11 @@ func (sp *ServicePlan) Format() string {
 
 	specDiff := specTable.String() + "\n"
 
-	// Render labels as nested blocks outside the table to not widen its attribute column. An existing service without
+	// Render the labels outside the table because they span multiple lines.
-	// an old spec only has new containers added to up-to-date ones so its labels didn't change.
+	if specChanged {
-	if sp.IsNewService || oldSpec != nil {
+		specDiff += formatLabelsDiff("labels:", old.Labels, sp.Spec.Labels, sp.IsNewService)
-		var oldLabels, oldContainerLabels map[string]string
+		specDiff += formatLabelsDiff("container labels:", old.Container.Labels, sp.Spec.Container.Labels,
-		if oldSpec != nil {
+			sp.IsNewService)
-			oldLabels, oldContainerLabels = oldSpec.Labels, oldSpec.Container.Labels
-		}
-		specDiff += formatLabelsDiff("labels:", oldLabels, sp.Spec.Labels, sp.IsNewService)
-		specDiff += formatLabelsDiff("container labels:", oldContainerLabels, sp.Spec.Container.Labels, sp.IsNewService)
 	}
 
 	treePrefix := tui.Faint.Render("  │ ")
@@ -288,6 +300,65 @@ func formatImageDiff(oldImage, newImage string) string {
 		tui.FormatImage(newImage, tui.Green)
 }
 
+// formatCaddyConfigDiff returns the modifier and the number of added and removed lines in the Caddy config. It omits
+// the config content because it can be large. It returns an empty value if the configs are the same.
+func formatCaddyConfigDiff(oldConfig, newConfig string, isNewService bool) (mod, value string) {
+	if oldConfig == newConfig {
+		return "", ""
+	}
+
+	// Count lines regardless of their order, which is enough for a summary and doesn't require a diff algorithm.
+	remaining := make(map[string]int)
+	var removed int
+	if oldConfig != "" {
+		for line := range strings.SplitSeq(oldConfig, "\n") {
+			remaining[line]++
+			removed++
+		}
+	}
+	var added int
+	if newConfig != "" {
+		for line := range strings.SplitSeq(newConfig, "\n") {
+			if remaining[line] > 0 {
+				remaining[line]--
+				removed--
+			} else {
+				added++
+			}
+		}
+	}
+
+	var parts []string
+	if added > 0 {
+		parts = append(parts, tui.Green.Render("+"+strconv.Itoa(added)))
+	}
+	if removed > 0 {
+		parts = append(parts, tui.Red.Render("-"+strconv.Itoa(removed)))
+	}
+
+	switch {
+	case added+removed == 0:
+		value = "changed"
+	case added+removed == 1:
+		value = parts[0] + " line"
+	default:
+		value = strings.Join(parts, " ") + " lines"
+	}
+
+	switch {
+	case isNewService:
+		mod = ""
+	case oldConfig == "":
+		mod = tui.Green.Render("+")
+	case newConfig == "":
+		mod = tui.Red.Render("-")
+	default:
+		mod = "~"
+	}
+
+	return mod, value
+}
+
 // formatLabelsDiff renders a block with the attribute name followed by a nested line for each added, changed, or
 // removed label, sorted by key. It returns an empty string if the labels are the same.
 func formatLabelsDiff(attribute string, oldLabels, newLabels map[string]string, isNewService bool) string {

pkg/client/deploy/deploy_test.go

@@ -83,6 +83,21 @@ func TestServicePlan_Format(t *testing.T) {
 		}
 	}
 
+	withCaddy := func(config string) api.ServiceSpec {
+		s := spec("nginx:1.25", nil, nil)
+		s.Caddy = &api.CaddySpec{Config: config}
+		return s
+	}
+	replaceFrom := func(old api.ServiceSpec) func(api.ServiceSpec) []operation.Operation {
+		return func(new api.ServiceSpec) []operation.Operation {
+			return []operation.Operation{&operation.ReplaceContainerOperation{
+				Spec:         new,
+				MachineName:  "machine-1",
+				OldContainer: newContainer(old),
+			}}
+		}
+	}
+
 	tests := []struct {
 		name         string
 		isNewService bool
@@ -90,6 +105,112 @@ func TestServicePlan_Format(t *testing.T) {
 		ops          func(spec api.ServiceSpec) []operation.Operation
 		expected     []string
 	}{
+		{
+			name: "caddy config lines added and removed",
+			spec: withCaddy("{\n\tstorage uncloud\n}"),
+			ops:  replaceFrom(withCaddy("{\n\tdebug\n\tauto_https off\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ ~ caddy config: +1 -2 lines",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config single line added",
+			spec: withCaddy("{\n\tstorage uncloud\n\tdebug\n}"),
+			ops:  replaceFrom(withCaddy("{\n\tstorage uncloud\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ ~ caddy config: +1 line",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config added to existing service",
+			spec: withCaddy("{\n\tstorage uncloud\n}"),
+			ops:  replaceFrom(spec("nginx:1.25", nil, nil)),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ + caddy config: +3 lines",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config removed",
+			spec: spec("nginx:1.25", nil, nil),
+			ops:  replaceFrom(withCaddy("{\n\tstorage uncloud\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ - caddy config: -3 lines",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config with only reordered lines",
+			spec: withCaddy("a {\n}\nb {\n}"),
+			ops:  replaceFrom(withCaddy("b {\n}\na {\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ ~ caddy config: changed",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config unchanged ignoring surrounding whitespace",
+			spec: withCaddy("{\n\tdebug\n}"),
+			ops:  replaceFrom(withCaddy("\n{\n\tdebug\n}\n")),
+			expected: []string{
+				"~ update service web",
+				"  │   image: nginx:1.25",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name:         "new service with caddy config",
+			isNewService: true,
+			spec:         withCaddy("{\n\tstorage uncloud\n}"),
+			ops: func(new api.ServiceSpec) []operation.Operation {
+				return []operation.Operation{&operation.RunContainerOperation{Spec: new, MachineName: "machine-1"}}
+			},
+			expected: []string{
+				"+ create service web",
+				"  │   image:        nginx:1.25",
+				"  │   caddy config: +3 lines",
+				"  │",
+				"  ╰── +   run container web on machine-1",
+			},
+		},
+		{
+			name:         "caddy config before replicas",
+			isNewService: true,
+			spec: func() api.ServiceSpec {
+				s := withCaddy("{\n\tstorage uncloud\n}")
+				s.Replicas = 3
+				return s
+			}(),
+			ops: func(new api.ServiceSpec) []operation.Operation {
+				return []operation.Operation{&operation.RunContainerOperation{Spec: new, MachineName: "machine-1"}}
+			},
+			expected: []string{
+				"+ create service web",
+				"  │   image:        nginx:1.25",
+				"  │   caddy config: +3 lines",
+				"  │   replicas:     3",
+				"  │",
+				"  ╰── +   run container web on machine-1",
+			},
+		},
 		{
 			name: "service labels added, changed, and removed in place",
 			spec: spec("nginx:1.25", map[string]string{"env": "prod", "team": "platform", "same": "1"}, nil),

test/e2e/service_test.go

@@ -319,6 +319,8 @@ func TestDeployment(t *testing.T) {
 		deployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{})
 		require.NoError(t, err)
 		deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
+		image := deployment.Spec.Container.Image
+		assert.Regexp(t, `^ghcr\.io/unlabs-dev/caddy-uncloud:0\.\d+\.\d+$`, image)
 
 		_, err = deployment.Run(ctx)
 		require.NoError(t, err)
@@ -327,15 +329,72 @@ func TestDeployment(t *testing.T) {
 		require.NoError(t, err)
 		assert.Len(t, svc.Containers, 3)
 		assertServiceMatchesSpec(t, svc, deployment.Spec)
+		assert.Equal(t, map[string]string{api.LabelCaddyManaged: ""}, svc.Spec().Labels)
 
-		ctr := svc.Containers[0].Container
+		var config api.CaddyConfig
-		assert.Regexp(t, `^caddy:2\.\d+\.\d+$`, ctr.Config.Image)
+		require.Eventually(t, func() bool {
-
+			config, err = cli.Caddy.Config(ctx, client.CaddyConfigOptions{})
-		config, err := cli.Caddy.Config(ctx, client.CaddyConfigOptions{})
+			return err == nil && strings.Contains(config.Caddyfile, "storage uncloud")
-		require.NoError(t, err)
+		}, 10*time.Second, 200*time.Millisecond, "expected the Caddyfile to enable cluster storage")
-
+		assert.Empty(t, config.LastReconciliationError)
 		assert.Contains(t, config.Caddyfile, "# Caddyfile autogenerated by Uncloud")
 		assert.Contains(t, config.Caddyfile, "handle /.uncloud-verify")
+
+		// Setting the running image as custom only changes the reserved service labels so it must not recreate
+		// the containers.
+		containerIDs := serviceContainerIDs(svc)
+		deployment, err = cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: image})
+		require.NoError(t, err)
+
+		_, err = deployment.Run(ctx)
+		require.NoError(t, err)
+
+		svc, err = cli.InspectService(ctx, client.CaddyServiceName)
+		require.NoError(t, err)
+		assert.True(t, containerIDs.Equal(serviceContainerIDs(svc)), "expected the containers to be updated in place")
+		assert.Equal(t, map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyImageCustom: ""}, svc.Spec().Labels)
+	})
+
+	t.Run("caddy current spec deployment only adds a container on a new machine", func(t *testing.T) {
+		t.Cleanup(func() {
+			err := cli.RemoveService(ctx, client.CaddyServiceName)
+			if !errors.Is(err, api.ErrNotFound) {
+				require.NoError(t, err)
+			}
+		})
+
+		deployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{})
+		require.NoError(t, err)
+		deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
+
+		_, err = deployment.Run(ctx)
+		require.NoError(t, err)
+
+		svc, err := cli.InspectService(ctx, client.CaddyServiceName)
+		require.NoError(t, err)
+		require.Len(t, svc.Containers, 3)
+
+		// Remove a container to make its machine look like a new one that doesn't run Caddy yet, as 'uc machine add' sees it.
+		removed := svc.Containers[0]
+		err = cli.RemoveContainer(ctx, svc.ID, removed.Container.ID, container.RemoveOptions{Force: true})
+		require.NoError(t, err)
+
+		svc, err = cli.InspectService(ctx, client.CaddyServiceName)
+		require.NoError(t, err)
+		require.Len(t, svc.Containers, 2)
+		containerIDs := serviceContainerIDs(svc)
+
+		_, err = cli.NewDeployment(svc.Spec(), nil).Run(ctx)
+		require.NoError(t, err)
+
+		svc, err = cli.InspectService(ctx, client.CaddyServiceName)
+		require.NoError(t, err)
+		assert.Len(t, svc.Containers, 3)
+		assert.Len(t, serviceMachines(svc).ToSlice(), 3, "expected 1 container on each machine")
+		assert.True(t, containerIDs.IsSubset(serviceContainerIDs(svc)),
+			"expected other containers to be left unchanged")
+		assertServiceMatchesSpec(t, svc, deployment.Spec)
+		assert.Equal(t, map[string]string{api.LabelCaddyManaged: ""}, svc.Spec().Labels)
 	})
 
 	t.Run("caddy with machine placement", func(t *testing.T) {
@@ -365,6 +424,11 @@ func TestDeployment(t *testing.T) {
 		assert.Equal(t, c.Machines[0].ID, svc.Containers[0].MachineID)
 		// initialContainerID := svc.Containers[0].Container.ID
 
+		// Deploying the current spec keeps the placement so it must not add containers on other machines.
+		plan, err := cli.NewDeployment(svc.Spec(), nil).Plan(ctx)
+		require.NoError(t, err)
+		assert.Empty(t, plan.Operations)
+
 		// Deploy to all machines without a placement constraint.
 		deployment, err = cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: image})
 		require.NoError(t, err)
@@ -447,7 +511,7 @@ func TestDeployment(t *testing.T) {
 myapp.example.com {
 	reverse_proxy 1.2.3.4:8000
 }`
-		caddyDeployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Config: caddyCaddyfile})
+		caddyDeployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Config: &caddyCaddyfile})
 		require.NoError(t, err)
 		caddyDeployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
 
@@ -1832,11 +1896,21 @@ func TestServiceLifecycle(t *testing.T) {
 		require.NoError(t, err)
 		assertContainerMatchesSpec(t, mc.Container, spec)
 
-		// Reserved service labels are rejected.
+		// Service labels with an invalid key format are rejected.
 		err = cli.UpdateContainer(ctx, serviceID, ctr.ID, api.UpdateContainerOptions{
-			ServiceLabels: &map[string]string{api.LabelManaged: ""},
+			ServiceLabels: &map[string]string{"my label": ""},
 		})
-		require.ErrorContains(t, err, "reserved prefix")
+		require.ErrorContains(t, err, "must not contain whitespace")
+
+		// The daemon accepts reserved service labels because Uncloud sets its own labels on services.
+		err = cli.UpdateContainer(ctx, serviceID, ctr.ID, api.UpdateContainerOptions{
+			ServiceLabels: &map[string]string{"uncloud.future": ""},
+		})
+		require.NoError(t, err)
+
+		mc, err = cli.InspectContainer(ctx, serviceID, ctr.ID)
+		require.NoError(t, err)
+		assert.Equal(t, map[string]string{"uncloud.future": ""}, mc.Container.ServiceSpec.Labels)
 
 		require.NoError(t, cli.RemoveContainer(ctx, serviceID, ctr.ID, container.RemoveOptions{Force: true}))
 		err = cli.UpdateContainer(ctx, serviceID, ctr.ID, api.UpdateContainerOptions{

website/docs/2-getting-started/2-deploy-demo-app.md

@@ -156,7 +156,7 @@ Waiting for the machine to be ready...
 Reserved cluster domain: sh8hsb.uncld.dev
 [+] Deploying service caddy 2/2
  ✔ Container caddy-d7uk on machine-dc3c     Running                                           11.1s
-   ✔ Image caddy:2.11.4 on machine-dc3c     Pulled                                             3.7s
+   ✔ Image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3 on machine-dc3c     Pulled                   3.7s
 
 Updating cluster domain records in Uncloud DNS to point to machines running caddy service...
 [+] Verifying internet access to caddy service 1/1
@@ -216,9 +216,9 @@ uc ls
 ```
 
 ```
-NAME         MODE         REPLICAS   IMAGE                          ENDPOINTS
+NAME         MODE         REPLICAS   IMAGE                                    ENDPOINTS
-caddy        global       1          caddy:2.11.4
+caddy        global       1          ghcr.io/unlabs-dev/caddy-uncloud:0.1.3
-excalidraw   replicated   1          excalidraw/excalidraw:latest   https://excalidraw.sh8hsb.uncld.dev → :80
+excalidraw   replicated   1          excalidraw/excalidraw:latest             https://excalidraw.sh8hsb.uncld.dev → :80
 ```
 
 You can see `caddy` service listed here. That's your reverse proxy, running as a regular Uncloud service.

website/docs/3-concepts/2-ingress/1-overview.md

@@ -25,10 +25,6 @@ When you [publish a service port](2-publishing-services.md), Uncloud automatical
 For advanced use cases, Uncloud allows to customise the Caddy config using the `x-caddy` extension in Compose files.
 See [Custom Caddy configuration](2-publishing-services.md#custom-caddy-configuration) for details.
 
-:::tip Shared TLS certificates
+Uncloud configures Caddy to use cluster storage by default. It shares TLS certificates and ACME challenge tokens across
-
-Uncloud also provides native cluster storage for Caddy. It shares TLS certificates and ACME challenge tokens across
 machines and coordinates certificate issuance. See [Cluster storage for Caddy](4-cluster-storage-for-caddy.md)
-to enable it.
+for details.
-
-:::

website/docs/9-cli-reference/uc_caddy_deploy.md

@@ -7,17 +7,59 @@ Deploy or upgrade Caddy reverse proxy across all machines in the cluster.
 Deploy or upgrade Caddy reverse proxy across all machines in the cluster.
 A rolling update is performed when updating existing containers to minimise disruption.
 
+By default, this deploys the latest version of the Uncloud Caddy image with a global config that enables cluster
+storage for TLS certificates. See https://github.com/unlabs-dev/caddy-uncloud for more details.
+
+Values set with --image, --caddyfile, and --machine are kept for later deployments. Pass an empty value to reset
+a flag to its default.
+
+If this command didn't deploy the current caddy service, for example, it was deployed with 'uc deploy'
+from a Compose file or an older version of uc, the deployment replaces its configuration.
+
+Requires Uncloud daemon 0.22.0 or newer on all machines.
+
 ```
 uc caddy deploy [flags]
 ```
 
+## Examples
+
+```
+  # Deploy Caddy or upgrade it to the latest version of the Uncloud Caddy image.
+  uc caddy deploy
+
+  # Deploy a custom image and keep it for later deployments.
+  uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3
+
+  # Go back to the latest version of the Uncloud Caddy image.
+  uc caddy deploy --image ""
+
+  # Deploy a custom image build and a custom global Caddy config and keep both for later deployments.
+  uc caddy deploy --image myorg/caddy-uncloud:latest --caddyfile global.Caddyfile
+
+  # Go back to the default global Caddy config but keep the custom image.
+  uc caddy deploy --caddyfile ""
+
+  # Deploy only to specific machines and keep them for later deployments.
+  uc caddy deploy --machine machine1,machine2
+
+  # Go back to deploying to all machines.
+  uc caddy deploy --machine ""
+```
+
 ## Options
 
 ```
       --caddyfile string   Path to a custom global Caddy config (Caddyfile) that will be prepended to the auto-generated Caddy config.
+                           An empty value resets it to the default config that enables cluster storage.
+                           (default is the current custom config or the default config)
   -h, --help               help for deploy
-      --image string       Caddy Docker image to deploy. (default caddy:LATEST_VERSION)
+      --image string       Caddy Docker image to deploy. It must include the Uncloud storage module if the Caddy config uses it.
-  -m, --machine strings    Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated list. (default is all machines)
+                           An empty value resets it to the latest version of the Uncloud Caddy image.
+                           (default is the current custom image or ghcr.io/unlabs-dev/caddy-uncloud:LATEST_VERSION)
+  -m, --machine strings    Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated list.
+                           An empty value resets it to all machines.
+                           (default is the current machines or all machines)
 ```
 
 ## Options inherited from parent commands