docs(releases): copy release notes from github to docs for v0.16.0-v0.21.0

README.md

@@ -44,8 +44,8 @@ complexity of Kubernetes.
   [Uncloud DNS](https://github.com/psviderski/uncloud-dns) service.
 * **Automatic HTTPS**: Built-in Caddy reverse proxy handles TLS certificate provisioning and renewal using Let's
   Encrypt.
-* **Cluster storage for TLS certificates**: Native cluster storage for Caddy shares certificates and ACME challenge
-  tokens across machines and coordinates issuance.
+* **Cluster storage for TLS certificates**: [Cluster storage](https://github.com/unlabs-dev/caddy-uncloud) for Caddy
+  shares certificates and ACME challenge tokens across machines and coordinates issuance.
 * **Docker-like CLI**: Familiar commands for managing both infrastructure and applications.
 * **Remote management**: Control your entire infrastructure through SSH access to any single machine in the cluster.
 

cmd/uc/caddy/cert/ls.go

@@ -17,6 +17,7 @@ import (
 	"github.com/psviderski/uncloud/internal/cli"
 	"github.com/psviderski/uncloud/internal/cli/completion"
 	"github.com/psviderski/uncloud/internal/cli/tui"
+	"github.com/psviderski/uncloud/internal/version"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client"
 	"github.com/spf13/cobra"
@@ -35,10 +36,13 @@ func NewListCommand() *cobra.Command {
 		Short:   "List certificates in cluster storage for Caddy.",
 		Long: `List certificates stored in the Uncloud cluster storage for Caddy.
 
-Caddy must use the Uncloud storage module (https://github.com/unlabs-dev/caddy-uncloud)
-configured with 'storage uncloud' in the global options for its certificates to appear here.
+Caddy uses cluster storage by default. Certificates don't appear here if Caddy uses local storage,
+for example, if it runs with a custom global config without 'storage uncloud' or was deployed with
+uc v0.21.0 or earlier and not upgraded.
 
-This inventory does not check whether Caddy currently serves or trusts a certificate.`,
+This inventory does not check whether Caddy currently serves or trusts a certificate.
+
+See ` + version.DocsURL + `/concepts/ingress/cluster-storage-for-caddy for details.`,
 		Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			uncli := cmd.Context().Value("cli").(*cli.CLI)

cmd/uc/caddy/deploy.go

@@ -16,6 +16,7 @@ import (
 	"github.com/psviderski/uncloud/internal/cli"
 	"github.com/psviderski/uncloud/internal/cli/completion"
 	"github.com/psviderski/uncloud/internal/cli/tui"
+	"github.com/psviderski/uncloud/internal/version"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client"
 	"github.com/spf13/cobra"
@@ -25,6 +26,10 @@ type deployOptions struct {
 	caddyfile string
 	image     string
 	machines  []string
+
+	caddyfileChanged bool
+	imageChanged     bool
+	machinesChanged  bool
 }
 
 func NewDeployCommand() *cobra.Command {
@@ -33,21 +38,62 @@ func NewDeployCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "deploy",
 		Short: "Deploy or upgrade Caddy reverse proxy across all machines in the cluster.",
-		Long: "Deploy or upgrade Caddy reverse proxy across all machines in the cluster.\n" +
-			"A rolling update is performed when updating existing containers to minimise disruption.",
+		Long: `Deploy or upgrade Caddy reverse proxy across all machines in the cluster.
+A rolling update is performed when updating existing containers to minimise disruption.
+
+By default, this deploys the latest version of the Uncloud Caddy image with a global config that enables cluster
+storage for TLS certificates. See https://github.com/unlabs-dev/caddy-uncloud for more details.
+
+Values set with --image, --caddyfile, and --machine are kept for later deployments. Pass an empty value to reset
+a flag to its default.
+
+If this command didn't deploy the current caddy service, for example, it was deployed with 'uc deploy'
+from a Compose file or an older version of uc, the deployment replaces its configuration.
+
+Requires Uncloud daemon ` + client.MinCaddyDaemonVersion + ` or newer on all machines.`,
+		Example: `  # Deploy Caddy or upgrade it to the latest version of the Uncloud Caddy image.
+  uc caddy deploy
+
+  # Deploy a custom image and keep it for later deployments.
+  uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3
+
+  # Go back to the latest version of the Uncloud Caddy image.
+  uc caddy deploy --image ""
+
+  # Deploy a custom image build and a custom global Caddy config and keep both for later deployments.
+  uc caddy deploy --image myorg/caddy-uncloud:latest --caddyfile global.Caddyfile
+
+  # Go back to the default global Caddy config but keep the custom image.
+  uc caddy deploy --caddyfile ""
+
+  # Deploy only to specific machines and keep them for later deployments.
+  uc caddy deploy --machine machine1,machine2
+
+  # Go back to deploying to all machines.
+  uc caddy deploy --machine ""`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			uncli := cmd.Context().Value("cli").(*cli.CLI)
+
+			opts.caddyfileChanged = cmd.Flag("caddyfile").Changed
+			opts.imageChanged = cmd.Flag("image").Changed
+			opts.machinesChanged = cmd.Flag("machine").Changed
+
 			return runDeploy(cmd.Context(), uncli, opts)
 		},
 	}
 
 	cmd.Flags().StringVar(&opts.caddyfile, "caddyfile", "",
-		"Path to a custom global Caddy config (Caddyfile) that will be prepended to the auto-generated Caddy config.")
+		"Path to a custom global Caddy config (Caddyfile) that will be prepended to the auto-generated Caddy config.\n"+
+			"An empty value resets it to the default config that enables cluster storage.\n"+
+			"(default is the current custom config or the default config)")
 	cmd.Flags().StringVar(&opts.image, "image", "",
-		"Caddy Docker image to deploy. (default caddy:LATEST_VERSION)")
+		"Caddy Docker image to deploy. It must include the Uncloud storage module if the Caddy config uses it.\n"+
+			"An empty value resets it to the latest version of the Uncloud Caddy image.\n"+
+			"(default is the current custom image or "+client.CaddyImage+":LATEST_VERSION)")
 	cmd.Flags().StringSliceVarP(&opts.machines, "machine", "m", nil,
-		"Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated "+
-			"list. (default is all machines)")
+		"Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated list.\n"+
+			"An empty value resets it to all machines.\n"+
+			"(default is the current machines or all machines)")
 
 	completion.MachinesFlag(cmd)
 
@@ -55,13 +101,13 @@ func NewDeployCommand() *cobra.Command {
 }
 
 func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
-	caddyfile := ""
+	config := ""
 	if opts.caddyfile != "" {
 		data, err := os.ReadFile(opts.caddyfile)
 		if err != nil {
 			return fmt.Errorf("read Caddyfile: %w", err)
 		}
-		caddyfile = strings.TrimSpace(string(data))
+		config = strings.TrimSpace(string(data))
 	}
 
 	clusterClient, err := uncli.ConnectCluster(ctx)
@@ -70,6 +116,10 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 	}
 	defer clusterClient.Close()
 
+	if err = clusterClient.Caddy.CheckDaemonVersions(ctx); err != nil {
+		return err
+	}
+
 	svc, err := clusterClient.InspectService(ctx, client.CaddyServiceName)
 	if err != nil {
 		if !errors.Is(err, api.ErrNotFound) {
@@ -100,26 +150,48 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 		}
 	}
 
-	if opts.image != "" {
-		fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(opts.image, tui.Green))
+	current := svc.Spec()
+	deployOpts := resolveDeploymentOptions(current, opts, config)
+	configChanged := deployOpts.ConfigOrDefault() != current.CaddyConfig()
+
+	if svc.ID != "" && configChanged {
+		printCaddyConfig("current global Caddy config:", current.CaddyConfig())
 	}
 
 	fmt.Println()
 	fmt.Println("Preparing a deployment plan...")
 
-	placement := api.Placement{
-		Machines: cli.ExpandCommaSeparatedValues(opts.machines),
-	}
-	d, err := clusterClient.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{
-		Image: opts.image, Config: caddyfile, Placement: placement,
-	})
+	d, err := clusterClient.Caddy.NewDeployment(ctx, deployOpts)
 	if err != nil {
 		return fmt.Errorf("create caddy deployment: %w", err)
 	}
+	target := d.Spec
 
-	if opts.image == "" {
-		fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(d.Spec.Container.Image,
-			tui.Green) + tui.Faint.Render(" (latest stable)"))
+	// The Uncloud Caddy image resolves to the latest version allowed by this CLI, which is older than the running image
+	// if a newer CLI deployed it. Continuing would downgrade the image and replace its default config.
+	if deployOpts.Image == "" && client.IsNewerCaddyImage(current.Container.Image, target.Container.Image) {
+		return fmt.Errorf("caddy service runs image '%s' which is newer than the latest version '%s' supported by "+
+			"this uc version: upgrade uc or specify the image with --image",
+			current.Container.Image, target.Container.Image)
+	}
+
+	imageNote := " (latest)"
+	if deployOpts.Image != "" {
+		imageNote = " (custom)"
+		if !opts.imageChanged {
+			imageNote = " (custom, use --image \"\" for the latest version)"
+		}
+	}
+	fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(target.Container.Image, tui.Green) +
+		tui.Faint.Render(imageNote))
+
+	if configChanged {
+		if deployOpts.Config != nil {
+			// The user provided this config so there is no need to print its content.
+			fmt.Println(tui.Faint.Render("target global Caddy config: ") + opts.caddyfile)
+		} else {
+			printCaddyConfig("target global Caddy config: (default)", deployOpts.ConfigOrDefault())
+		}
 	}
 
 	plan, err := d.Plan(ctx)
@@ -131,7 +203,7 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 		fmt.Printf("%s service is up to date.\n", client.CaddyServiceName)
 	} else {
 		if svc.ID == "" {
-			if len(opts.machines) > 0 {
+			if len(deployOpts.Placement.Machines) > 0 {
 				fmt.Println("This will run a Caddy container on each selected machine.")
 			} else {
 				fmt.Println("This will run a Caddy container on each machine.")
@@ -163,6 +235,14 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 		fmt.Println(summary)
 		fmt.Println()
 
+		warnings := deploymentWarnings(current, target, opts)
+		for _, warning := range warnings {
+			tui.PrintWarning(warning)
+		}
+		if len(warnings) > 0 {
+			fmt.Println()
+		}
+
 		title := "Proceed with deployment?"
 		// Include the direct connection or context name in the confirmation prompt to avoid accidentally
 		// deploying to the wrong cluster.
@@ -195,6 +275,98 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 	return UpdateDomainRecords(ctx, clusterClient, uncli.ProgressOut())
 }
 
+// resolveDeploymentOptions returns the deployment options for the flags and the current service spec, which is zero if
+// the service doesn't exist. Each flag works on its own: a specified flag overrides the value and an empty one resets
+// it to the default. A flag that isn't specified keeps the explicitly set value.
+func resolveDeploymentOptions(
+	current api.ServiceSpec, opts deployOptions, config string,
+) client.CaddyDeploymentOptions {
+	var resolved client.CaddyDeploymentOptions
+
+	if opts.imageChanged {
+		resolved.Image = opts.image
+	} else if _, custom := current.Labels[api.LabelCaddyImageCustom]; custom {
+		resolved.Image = current.Container.Image
+	}
+
+	if opts.caddyfileChanged {
+		if opts.caddyfile != "" {
+			resolved.Config = new(config)
+		}
+	} else if _, custom := current.Labels[api.LabelCaddyConfigCustom]; custom {
+		resolved.Config = new(current.CaddyConfig())
+	}
+
+	if opts.machinesChanged {
+		resolved.Placement.Machines = cli.ExpandCommaSeparatedValues(opts.machines)
+	} else if _, managed := current.Labels[api.LabelCaddyManaged]; managed {
+		resolved.Placement = current.Placement
+	}
+
+	return resolved
+}
+
+// deploymentWarnings returns the consequences of deploying the target spec over the current one, which is zero if
+// the service doesn't exist.
+func deploymentWarnings(current, target api.ServiceSpec, opts deployOptions) []string {
+	var warnings []string
+	exists := current.Name != ""
+
+	if _, managed := current.Labels[api.LabelCaddyManaged]; exists && !managed {
+		warnings = append(warnings, "'uc caddy deploy' doesn't manage this caddy service yet, so this deployment "+
+			"replaces its configuration, including any custom global Caddy config.\n"+
+			"If you manage it with a Compose file, cancel and deploy with 'uc deploy' instead.")
+	}
+
+	currentStorage := client.CaddyStorageModule(current.CaddyConfig())
+	targetStorage := client.CaddyStorageModule(target.CaddyConfig())
+	if exists && currentStorage != targetStorage {
+		warning := fmt.Sprintf("Caddy certificate storage changes from %s to %s.\n"+
+			"Caddy doesn't automatically migrate certificates, so all of them will be reissued and HTTPS may fail "+
+			"for those domains until that finishes.\n"+
+			"To migrate them first, see %s/concepts/ingress/cluster-storage-for-caddy",
+			caddyStorageName(currentStorage), caddyStorageName(targetStorage), version.DocsURL)
+		if !opts.caddyfileChanged {
+			warning += "\nTo keep the current storage, specify a custom Caddyfile that uses it with --caddyfile."
+		}
+		warnings = append(warnings, warning)
+	}
+
+	imageOrConfigChanged := current.Container.Image != target.Container.Image ||
+		current.CaddyConfig() != target.CaddyConfig()
+	if targetStorage == client.CaddyStorageUncloud && imageOrConfigChanged &&
+		!client.IsDefaultCaddyImage(target.Container.Image) {
+		warnings = append(warnings, fmt.Sprintf("Image '%s' must include the Uncloud storage module to use "+
+			"'storage uncloud'. Otherwise, Caddy won't start or apply the new config.\n"+
+			"Check 'uc caddy config' after the deployment for errors.", target.Container.Image))
+	}
+
+	return warnings
+}
+
+// caddyStorageName returns a user-friendly name of the Caddy storage module.
+func caddyStorageName(module string) string {
+	switch module {
+	case client.CaddyStorageFileSystem:
+		return "local"
+	case client.CaddyStorageUncloud:
+		return "cluster"
+	default:
+		return "'" + module + "'"
+	}
+}
+
+// printCaddyConfig prints the title and the indented config or "none" if it's empty.
+func printCaddyConfig(title, config string) {
+	if config == "" {
+		lipgloss.Println(tui.Faint.Render(title) + " none")
+		return
+	}
+
+	lipgloss.Println(tui.Faint.Render(title))
+	fmt.Println(config)
+}
+
 func UpdateDomainRecords(ctx context.Context, clusterClient *client.Client, progressOut *streams.Out) error {
 	domain, err := clusterClient.GetDomain(ctx)
 	if err != nil {

cmd/uc/caddy/deploy_test.go

@@ -0,0 +1,263 @@
+package caddy
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/psviderski/uncloud/pkg/client"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestResolveDeploymentOptions(t *testing.T) {
+	t.Parallel()
+
+	const (
+		customImage  = "example.com/caddy:1.2.3"
+		customConfig = "{\n\tdebug\n}"
+	)
+	machines := api.Placement{Machines: []string{"m1", "m2"}}
+
+	// A service with a custom image, config, and machines that has the given labels.
+	service := func(labels ...string) api.ServiceSpec {
+		spec := api.ServiceSpec{
+			Name:      client.CaddyServiceName,
+			Caddy:     &api.CaddySpec{Config: customConfig},
+			Container: api.ContainerSpec{Image: customImage},
+			Placement: machines,
+		}
+		if len(labels) > 0 {
+			spec.Labels = make(map[string]string, len(labels))
+			for _, label := range labels {
+				spec.Labels[label] = ""
+			}
+		}
+		return spec
+	}
+
+	tests := []struct {
+		name    string
+		current api.ServiceSpec
+		opts    deployOptions
+		config  string
+		want    client.CaddyDeploymentOptions
+	}{
+		{
+			name: "no service gets the defaults",
+		},
+		{
+			name:    "service that isn't managed gets the defaults",
+			current: service(),
+		},
+		{
+			name:    "managed service follows the default image and config and keeps machines",
+			current: service(api.LabelCaddyManaged),
+			want:    client.CaddyDeploymentOptions{Placement: machines},
+		},
+		{
+			name:    "custom image is kept and config follows the default",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom),
+			want:    client.CaddyDeploymentOptions{Image: customImage, Placement: machines},
+		},
+		{
+			name:    "custom config is kept and image follows the latest version",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyConfigCustom),
+			want:    client.CaddyDeploymentOptions{Config: new(customConfig), Placement: machines},
+		},
+		{
+			name:    "custom image and config are kept",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom, api.LabelCaddyConfigCustom),
+			want:    client.CaddyDeploymentOptions{Image: customImage, Config: new(customConfig), Placement: machines},
+		},
+		{
+			name: "custom empty config is kept",
+			current: api.ServiceSpec{
+				Name:   client.CaddyServiceName,
+				Labels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
+			},
+			want: client.CaddyDeploymentOptions{Config: new("")},
+		},
+		{
+			name:    "image flag overrides the current image",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom),
+			opts:    deployOptions{image: "caddy:2", imageChanged: true},
+			want:    client.CaddyDeploymentOptions{Image: "caddy:2", Placement: machines},
+		},
+		{
+			name:    "empty image flag resets to the latest version",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom),
+			opts:    deployOptions{imageChanged: true},
+			want:    client.CaddyDeploymentOptions{Placement: machines},
+		},
+		{
+			name:    "caddyfile flag overrides the current config",
+			current: service(api.LabelCaddyManaged),
+			opts:    deployOptions{caddyfile: "global.Caddyfile", caddyfileChanged: true},
+			config:  "{\n\tstorage redis\n}",
+			want:    client.CaddyDeploymentOptions{Config: new("{\n\tstorage redis\n}"), Placement: machines},
+		},
+		{
+			name:    "caddyfile flag with an empty file sets a custom empty config",
+			current: service(api.LabelCaddyManaged),
+			opts:    deployOptions{caddyfile: "empty.Caddyfile", caddyfileChanged: true},
+			want:    client.CaddyDeploymentOptions{Config: new(""), Placement: machines},
+		},
+		{
+			name:    "empty caddyfile flag resets to the default config",
+			current: service(api.LabelCaddyManaged, api.LabelCaddyConfigCustom),
+			opts:    deployOptions{caddyfileChanged: true},
+			want:    client.CaddyDeploymentOptions{Placement: machines},
+		},
+		{
+			name:    "machine flag overrides the current machines",
+			current: service(api.LabelCaddyManaged),
+			opts:    deployOptions{machines: []string{"m3,m4", "m5"}, machinesChanged: true},
+			want:    client.CaddyDeploymentOptions{Placement: api.Placement{Machines: []string{"m3", "m4", "m5"}}},
+		},
+		{
+			name:    "empty machine flag resets to all machines",
+			current: service(api.LabelCaddyManaged),
+			opts:    deployOptions{machines: []string{}, machinesChanged: true},
+		},
+		{
+			name:    "flags apply to a service that isn't managed",
+			current: service(),
+			opts: deployOptions{
+				image: "caddy:2", imageChanged: true,
+				caddyfile: "global.Caddyfile", caddyfileChanged: true,
+				machines: []string{"m3"}, machinesChanged: true,
+			},
+			config: customConfig,
+			want: client.CaddyDeploymentOptions{
+				Image:     "caddy:2",
+				Config:    new(customConfig),
+				Placement: api.Placement{Machines: []string{"m3"}},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, resolveDeploymentOptions(tt.current, tt.opts, tt.config))
+		})
+	}
+}
+
+func TestDeploymentWarnings(t *testing.T) {
+	t.Parallel()
+
+	const (
+		defaultImage = client.CaddyImage + ":0.1.3"
+		customImage  = "example.com/caddy:1.2.3"
+		localConfig  = "{\n\tdebug\n}"
+		override     = "doesn't manage this caddy service yet"
+		storage      = "certificate storage changes from"
+		keepStorage  = "To keep the current storage"
+		module       = "must include the Uncloud storage module"
+	)
+	managed := map[string]string{api.LabelCaddyManaged: ""}
+
+	service := func(image, config string, labels map[string]string) api.ServiceSpec {
+		spec := api.ServiceSpec{
+			Name:      client.CaddyServiceName,
+			Container: api.ContainerSpec{Image: image},
+			Labels:    labels,
+		}
+		if config != "" {
+			spec.Caddy = &api.CaddySpec{Config: config}
+		}
+		return spec
+	}
+
+	tests := []struct {
+		name    string
+		current api.ServiceSpec
+		target  api.ServiceSpec
+		opts    deployOptions
+		want    []string
+		notWant []string
+	}{
+		{
+			name:    "new service with defaults has no warnings",
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			notWant: []string{override, storage, module},
+		},
+		{
+			name:    "new service with a custom image",
+			target:  service(customImage, client.DefaultCaddyConfig, managed),
+			want:    []string{module},
+			notWant: []string{override, storage},
+		},
+		{
+			name:    "service that isn't managed moves to cluster storage",
+			current: service("caddy:2.11.4", "", nil),
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			want:    []string{override, "from local to cluster", keepStorage},
+			notWant: []string{module},
+		},
+		{
+			name:    "service that isn't managed already uses cluster storage",
+			current: service(defaultImage, client.DefaultCaddyConfig, nil),
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			want:    []string{override},
+			notWant: []string{storage, module},
+		},
+		{
+			name:    "managed service without changes has no warnings",
+			current: service(defaultImage, client.DefaultCaddyConfig, managed),
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			notWant: []string{override, storage, module},
+		},
+		{
+			name:    "managed service upgrades the default image",
+			current: service(client.CaddyImage+":0.1.2", client.DefaultCaddyConfig, managed),
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			notWant: []string{override, storage, module},
+		},
+		{
+			name:    "custom caddyfile moves to local storage",
+			current: service(defaultImage, client.DefaultCaddyConfig, managed),
+			target:  service(defaultImage, localConfig, managed),
+			opts:    deployOptions{caddyfile: "global.Caddyfile", caddyfileChanged: true},
+			want:    []string{"from cluster to local"},
+			notWant: []string{override, keepStorage, module},
+		},
+		{
+			name:    "other storage module is named",
+			current: service(defaultImage, "{\n\tstorage redis\n}", managed),
+			target:  service(defaultImage, client.DefaultCaddyConfig, managed),
+			want:    []string{"from 'redis' to cluster", keepStorage},
+		},
+		{
+			name:    "custom image is unchanged with the same config",
+			current: service(customImage, client.DefaultCaddyConfig, managed),
+			target:  service(customImage, client.DefaultCaddyConfig, managed),
+			notWant: []string{override, storage, module},
+		},
+		{
+			name:    "custom image receives a changed default config",
+			current: service(customImage, "{\n\tstorage uncloud\n\tdebug\n}", managed),
+			target:  service(customImage, client.DefaultCaddyConfig, managed),
+			want:    []string{module},
+			notWant: []string{override, storage},
+		},
+		{
+			name:    "custom image with local storage",
+			current: service(defaultImage, localConfig, managed),
+			target:  service(customImage, localConfig, managed),
+			notWant: []string{override, storage, module},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			warnings := strings.Join(deploymentWarnings(tt.current, tt.target, tt.opts), "\n")
+			for _, want := range tt.want {
+				assert.Contains(t, warnings, want)
+			}
+			for _, notWant := range tt.notWant {
+				assert.NotContains(t, warnings, notWant)
+			}
+		})
+	}
+}

cmd/uc/machine/add.go

@@ -187,14 +187,12 @@ func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine,
 	}
 	fmt.Println("Machine joined the cluster.")
 
-	// TODO: scale the existing Caddy service to the new machine instead of running a new deployment
-	//  that may cause a small downtime.
-	// Deploy a Caddy service container to the added machine. If caddy service is already deployed on other machines,
-	// use the deployed image version.
+	// Deploy the current caddy service spec to run a container on the added machine. This keeps the service
+	// configuration however it was deployed and shouldn't change the containers on other machines if specs on them
+	// haven't diverged.
 	// NOTE: We use the cluster client to inspect and scale the Caddy service because the newly added machine may have
 	// issues accessing the API of existing machines in the cluster.
 	// See the issue for more details: https://github.com/psviderski/uncloud/issues/65.
-	caddyImage := ""
 	caddySvc, err := clusterClient.InspectService(ctx, client.CaddyServiceName)
 	if err != nil {
 		if errors.Is(err, api.ErrNotFound) {
@@ -203,26 +201,10 @@ func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine,
 		}
 		return fmt.Errorf("inspect caddy service: %w", err)
 	}
-	caddyImage = caddySvc.Containers[0].Container.Config.Image
-	// Find the latest created container and use its image.
-	var latestCreated time.Time
-	for _, c := range caddySvc.Containers[1:] {
-		created, err := time.Parse(time.RFC3339Nano, c.Container.Created)
-		if err != nil {
-			continue
-		}
-		if created.After(latestCreated) {
-			latestCreated = created
-			caddyImage = c.Container.Config.Image
-		}
-	}
 
 	fmt.Println()
 	fmt.Println("Preparing Caddy deployment...")
-	d, err := clusterClient.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: caddyImage})
-	if err != nil {
-		return fmt.Errorf("create caddy deployment: %w", err)
-	}
+	d := clusterClient.NewDeployment(caddySvc.Spec(), nil)
 
 	plan, err := d.Plan(ctx)
 	if err != nil {

cmd/uc/service/run.go

@@ -106,11 +106,12 @@ func NewRunCommand(groupID string) *cobra.Command {
 	cmd.Flags().StringSliceVarP(&opts.publish, "publish", "p", nil,
 		"Publish a service port to make it accessible outside the cluster. Can be specified multiple times.\n"+
 			"Format: [hostname:]container_port[/protocol] or [host_ip|host_prefix:]host_port:container_port[/protocol]@host\n"+
-			"Supported protocols: tcp, udp, http, https (default is tcp). If a hostname for http(s) port is not specified\n"+
-			"and a cluster domain is reserved, service-name.cluster-domain will be used as the hostname.\n"+
+			"Supported protocols: tcp, udp, http, https. The protocol defaults to https if a hostname is specified,\n"+
+			"otherwise tcp. http(s) ports without a hostname use 'service-name.cluster-domain' as the default hostname\n"+
+			"if a cluster domain is reserved.\n"+
 			"Examples:\n"+
 			"  -p 8080/https                       Publish port 8080 as HTTPS via reverse proxy with default service-name.cluster-domain hostname\n"+
-			"  -p app.example.com:8080/https       Publish port 8080 as HTTPS via reverse proxy with custom hostname\n"+
+			"  -p app.example.com:8080             Publish port 8080 as HTTPS via reverse proxy with custom hostname\n"+
 			// TODO: add support for publishing L4 tcp/udp ports.
 			//"  -p 9000:8080                   Publish port 8080 as TCP port 9000 via reverse proxy\n"+
 			"  -p 53:5353/udp@host                 Bind UDP port 5353 to host port 53\n"+
@@ -208,6 +209,9 @@ func prepareServiceSpec(opts runOptions) (api.ServiceSpec, error) {
 	if err != nil {
 		return spec, err
 	}
+	if err = api.ValidateLabels(labels); err != nil {
+		return spec, fmt.Errorf("invalid service labels: %w", err)
+	}
 	containerLabels, err := parseLabels(opts.containerLabels)
 	if err != nil {
 		return spec, err

internal/machine/docker/service.go

@@ -166,7 +166,7 @@ func (s *Service) InspectServiceContainer(ctx context.Context, nameOrID string)
 // It applies the CPU and memory limits to the container and updates the service spec stored for the container.
 func (s *Service) UpdateServiceContainer(ctx context.Context, id string, opts api.UpdateContainerOptions) error {
 	if opts.ServiceLabels != nil {
-		if err := api.ValidateLabels(*opts.ServiceLabels); err != nil {
+		if err := api.ValidateLabelKeys(*opts.ServiceLabels); err != nil {
 			return errdefs.ErrInvalidArgument.WithMessage(fmt.Sprintf("invalid service labels: %v", err))
 		}
 	}

pkg/api/caddy.go

@@ -5,6 +5,15 @@ import (
 	"time"
 )
 
+const (
+	// LabelCaddyManaged marks the caddy service as managed by 'uc caddy deploy'.
+	LabelCaddyManaged = LabelReservedPrefix + "caddy.managed"
+	// LabelCaddyImageCustom marks the caddy image as explicitly set instead of following the latest default image.
+	LabelCaddyImageCustom = LabelReservedPrefix + "caddy.image.custom"
+	// LabelCaddyConfigCustom marks the global Caddy config as explicitly set instead of following the default config.
+	LabelCaddyConfigCustom = LabelReservedPrefix + "caddy.config.custom"
+)
+
 // CaddyConfig is the saved Caddy configuration on a machine.
 type CaddyConfig struct {
 	Caddyfile string

pkg/api/service.go

@@ -78,8 +78,8 @@ type ServiceSpec struct {
 	Configs []ConfigSpec
 	// Container defines the desired state of each container in the service.
 	Container ContainerSpec
-	// Labels are user-defined key-value metadata attached to the service. Unlike container labels, they're not
-	// applied to Docker containers. Keys must not use the reserved "uncloud." prefix.
+	// Labels are key-value metadata attached to the service. Unlike container labels, they're not applied to Docker
+	// containers. The "uncloud." key prefix is reserved for labels set by Uncloud.
 	Labels map[string]string
 	// Mode is the replication mode of the service. Default is ServiceModeReplicated if empty.
 	Mode string
@@ -161,7 +161,9 @@ func (s *ServiceSpec) Validate() error {
 	if err := s.Container.Validate(); err != nil {
 		return err
 	}
-	if err := ValidateLabels(s.Labels); err != nil {
+	// We want to allow reserved uncloud. labels in the service spec, because we can't distinguish between user-defined
+	// in a Compose file and those set by a uc command.
+	if err := ValidateLabelKeys(s.Labels); err != nil {
 		return fmt.Errorf("invalid service labels: %w", err)
 	}
 
@@ -444,9 +446,8 @@ func (s *ContainerSpec) Clone() ContainerSpec {
 	return spec
 }
 
-// ValidateLabels checks that label keys are non-empty, don't contain whitespace, and don't use the reserved Uncloud
-// prefixes.
-func ValidateLabels(labels map[string]string) error {
+// ValidateLabelKeys checks that label keys are non-empty and don't contain whitespace.
+func ValidateLabelKeys(labels map[string]string) error {
 	for k := range labels {
 		if strings.TrimSpace(k) == "" {
 			return fmt.Errorf("label key must not be empty")
@@ -454,6 +455,17 @@ func ValidateLabels(labels map[string]string) error {
 		if strings.ContainsFunc(k, unicode.IsSpace) {
 			return fmt.Errorf("label '%s' must not contain whitespace", k)
 		}
+	}
+	return nil
+}
+
+// ValidateLabels checks that label keys are valid and don't use the reserved Uncloud prefixes. Use it for labels
+// provided by users and for container labels.
+func ValidateLabels(labels map[string]string) error {
+	if err := ValidateLabelKeys(labels); err != nil {
+		return err
+	}
+	for k := range labels {
 		if strings.HasPrefix(k, LabelReservedPrefix) || strings.HasPrefix(k, LabelDaemonReservedPrefix) {
 			return fmt.Errorf("label '%s' cannot use the reserved prefix '%s' or '%s'",
 				k, LabelReservedPrefix, LabelDaemonReservedPrefix)

pkg/api/service_test.go

@@ -82,8 +82,6 @@ func TestServiceSpec_Validate_Labels(t *testing.T) {
 		{name: "trailing space in key", labels: map[string]string{"tier ": "bar"}, wantErr: "must not contain whitespace"},
 		{name: "tab in key", labels: map[string]string{"my\tlabel": "bar"}, wantErr: "must not contain whitespace"},
 		{name: "newline in key", labels: map[string]string{"my\nlabel": "bar"}, wantErr: "must not contain whitespace"},
-		{name: "reserved prefix", labels: map[string]string{LabelManaged: ""}, wantErr: "reserved prefix"},
-		{name: "reserved daemon prefix", labels: map[string]string{"uncloudd.foo": "bar"}, wantErr: "reserved prefix"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name+"/service", func(t *testing.T) {
@@ -111,6 +109,38 @@ func TestServiceSpec_Validate_Labels(t *testing.T) {
 	}
 }
 
+func TestServiceSpec_Validate_ReservedLabels(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name   string
+		labels map[string]string
+	}{
+		{name: "reserved prefix", labels: map[string]string{LabelManaged: ""}},
+		{name: "reserved daemon prefix", labels: map[string]string{"uncloudd.foo": "bar"}},
+		{name: "managed service", labels: map[string]string{
+			LabelReservedPrefix + "service.managed": "",
+			LabelReservedPrefix + "custom.key":      "value",
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			// Uncloud sets its own labels on services, so a service spec can have them.
+			spec := ServiceSpec{Container: ContainerSpec{Image: "nginx:latest"}, Labels: tt.labels}
+			require.NoError(t, spec.Validate())
+
+			// Container labels and labels provided by users can't use the reserved prefixes.
+			spec = ServiceSpec{Container: ContainerSpec{Image: "nginx:latest", Labels: tt.labels}}
+			err := spec.Validate()
+			require.ErrorContains(t, err, "invalid container labels")
+			require.ErrorContains(t, err, "reserved prefix")
+			require.ErrorContains(t, ValidateLabels(tt.labels), "reserved prefix")
+		})
+	}
+}
+
 func TestServiceSpec_Validate_CaddyAndPorts(t *testing.T) {
 	tests := []struct {
 		name    string

pkg/client/caddy.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/Masterminds/semver"
@@ -11,6 +12,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/psviderski/uncloud/api/pb"
+	"github.com/psviderski/uncloud/internal/version"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client/deploy"
 	"google.golang.org/protobuf/types/known/emptypb"
@@ -18,11 +20,33 @@ import (
 
 const (
 	CaddyServiceName = "caddy"
-	// CaddyImage is the official Caddy Docker image on Docker Hub: https://hub.docker.com/_/caddy
-	CaddyImage = "caddy"
+	// CaddyImage is the Uncloud Caddy image that is deployed by default. It includes the Uncloud storage module for
+	// sharing TLS certificates across machines: https://github.com/unlabs-dev/caddy-uncloud
+	CaddyImage = "ghcr.io/unlabs-dev/caddy-uncloud"
+	// DefaultCaddyConfig is the default global Caddyfile. It must be compatible with all versions of CaddyImage allowed
+	// by caddyImageConstraint.
+	DefaultCaddyConfig = "{\n\tstorage uncloud\n}"
+	// CaddyStorageFileSystem is the name of the default Caddy storage module that stores TLS assets locally.
+	CaddyStorageFileSystem = "file_system"
+	// CaddyStorageUncloud is the name of the Caddy storage module that stores TLS assets in the cluster.
+	CaddyStorageUncloud = "uncloud"
+	// MinCaddyDaemonVersion is the minimum daemon version required for 'uc caddy deploy'. Older daemons don't store
+	// service labels that mark the service as managed by this command. Distributed locks were introduced in 0.21.0,
+	// which is the minimum required by the cluster storage even on machines that don't run the Caddy service.
+	MinCaddyDaemonVersion = "0.22.0"
+
+	// officialCaddyImage is the official Caddy Docker image on Docker Hub: https://hub.docker.com/_/caddy
+	officialCaddyImage = "docker.io/library/caddy"
 )
 
-var caddyImageTagRegex = regexp.MustCompile(`^2\.\d+\.\d+$`)
+var (
+	// caddyImageConstraint limits CaddyImage to the major version that is known to be compatible with
+	// DefaultCaddyConfig.
+	caddyImageConstraint = mustNewConstraint("<1.0.0")
+	// caddyImageTagRegex matches the exact version tags of CaddyImage, skipping floating tags and image variants.
+	caddyImageTagRegex = regexp.MustCompile(`^\d+\.\d+\.\d+$`)
+	caddyStorageRegex  = regexp.MustCompile(`(?m)^[ \t]*storage[ \t]+(\S+)`)
+)
 
 // CaddyClient provides Caddy operations over its parent Client's connection.
 // The parent client owns the connection and must be used to close it.
@@ -64,19 +88,28 @@ func (c *CaddyClient) Config(ctx context.Context, opts CaddyConfigOptions) (api.
 	return config, nil
 }
 
-// CaddyDeploymentOptions configures a Caddy reverse proxy deployment.
+// CaddyDeploymentOptions configures a Caddy reverse proxy deployment. The zero value deploys the latest version of
+// CaddyImage with DefaultCaddyConfig to all machines.
 type CaddyDeploymentOptions struct {
-	// Image defaults to the latest stable 2.x.x official Caddy image.
-	// Custom images must include curl and start Caddy with /etc/caddy/Caddyfile.
+	// Image defaults to the latest version of CaddyImage. A non-empty value is marked as custom.
+	// Custom images must include curl and start Caddy with /etc/caddy/Caddyfile. They must also include the Uncloud
+	// storage module if the config uses 'storage uncloud'.
 	Image string
-	// Config contains an optional global Caddyfile.
-	Config    string
+	// Config is the global Caddyfile. Nil means DefaultCaddyConfig. Any other value, even empty, is marked as custom.
+	Config    *string
 	Placement api.Placement
 }
 
-// NewDeployment creates a new deployment for a Caddy reverse proxy service.
-// The service is deployed in global mode to all machines in the cluster. If the image is not provided, the latest
-// version of the official Caddy Docker image is used.
+// ConfigOrDefault returns the global Caddyfile of the deployment.
+func (o CaddyDeploymentOptions) ConfigOrDefault() string {
+	if o.Config != nil {
+		return *o.Config
+	}
+	return DefaultCaddyConfig
+}
+
+// NewDeployment creates a new managed deployment for a Caddy reverse proxy service.
+// The service is deployed in global mode to all machines in the cluster or the machines set in the placement.
 func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOptions) (*deploy.Deployment, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
@@ -90,6 +123,10 @@ func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOpt
 
 		image = reference.FamiliarString(latest)
 	}
+	config := opts.ConfigOrDefault()
+	if err := validateCaddyImageStorage(image, config); err != nil {
+		return nil, err
+	}
 
 	spec := api.ServiceSpec{
 		Container: api.ContainerSpec{
@@ -186,17 +223,119 @@ func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOpt
 		},
 	}
 
-	if opts.Config != "" {
+	if config != "" {
 		spec.Caddy = &api.CaddySpec{
-			Config: opts.Config,
+			Config: config,
 		}
 	}
 
+	spec.Labels = map[string]string{api.LabelCaddyManaged: ""}
+	if opts.Image != "" {
+		spec.Labels[api.LabelCaddyImageCustom] = ""
+	}
+	if opts.Config != nil {
+		spec.Labels[api.LabelCaddyConfigCustom] = ""
+	}
+
 	return c.client.NewDeployment(spec, nil), nil
 }
 
-// latestCaddyImage returns the latest image of the official Caddy Docker image on Docker Hub.
-// The latest image is determined by the latest version tag 2.x.x.
+// CheckDaemonVersions returns an error naming the available machines whose daemons are older than
+// MinCaddyDaemonVersion. Machines that are down are skipped because a deployment doesn't touch them.
+func (c *CaddyClient) CheckDaemonVersions(ctx context.Context) error {
+	machines, err := c.client.ListMachines(ctx, &api.MachineFilter{Available: true})
+	if err != nil {
+		return fmt.Errorf("list machines: %w", err)
+	}
+
+	if outdated := outdatedCaddyDaemons(machines); len(outdated) > 0 {
+		return fmt.Errorf("deploying Caddy requires Uncloud daemon %s or newer on all machines, upgrade these "+
+			"machines and try again: %s. See https://github.com/psviderski/uncloud/releases/latest",
+			MinCaddyDaemonVersion, strings.Join(outdated, ", "))
+	}
+	return nil
+}
+
+// outdatedCaddyDaemons returns descriptions of the machines with outdated daemons. Prereleases and development builds
+// aren't considered outdated because their versions don't show whether they're older than a stable release.
+func outdatedCaddyDaemons(machines api.MachineMembersList) []string {
+	minVersion := semver.MustParse(MinCaddyDaemonVersion)
+
+	var outdated []string
+	for _, m := range machines {
+		v, err := semver.NewVersion(m.Machine.DaemonVersion)
+		if err != nil || v.Prerelease() != "" {
+			continue
+		}
+		if v.LessThan(minVersion) {
+			outdated = append(outdated, fmt.Sprintf("%s (%s)", m.Machine.Name, m.Machine.DaemonVersion))
+		}
+	}
+
+	return outdated
+}
+
+// CaddyStorageModule returns the name of the storage module set in the global options of a Caddyfile.
+func CaddyStorageModule(config string) string {
+	if match := caddyStorageRegex.FindStringSubmatch(config); match != nil {
+		return match[1]
+	}
+	return CaddyStorageFileSystem
+}
+
+// IsDefaultCaddyImage returns whether the image is any version of CaddyImage.
+func IsDefaultCaddyImage(image string) bool {
+	return imageName(image) == CaddyImage
+}
+
+// IsNewerCaddyImage returns whether the image is a newer version of CaddyImage than the other image.
+// It returns false if any of the images isn't a version of CaddyImage.
+func IsNewerCaddyImage(image, other string) bool {
+	version, otherVersion := caddyImageVersion(image), caddyImageVersion(other)
+	return version != nil && otherVersion != nil && version.GreaterThan(otherVersion)
+}
+
+// caddyImageVersion returns the version in the image tag if the image is a version of CaddyImage, or nil otherwise.
+func caddyImageVersion(image string) *semver.Version {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil || named.Name() != CaddyImage {
+		return nil
+	}
+	tagged, ok := named.(reference.Tagged)
+	if !ok {
+		return nil
+	}
+	version, err := semver.NewVersion(tagged.Tag())
+	if err != nil {
+		return nil
+	}
+
+	return version
+}
+
+// imageName returns the normalised name of the image without its tag or digest.
+// It returns an empty string if the image reference is invalid.
+func imageName(image string) string {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return ""
+	}
+	return named.Name()
+}
+
+// validateCaddyImageStorage rejects the official Caddy image with a config that uses the Uncloud storage module
+// because it doesn't include this module. Otherwise, a new container could start from the previously saved Caddyfile
+// and pass its health check, but its routes would stop updating as Caddy would reject the new config.
+func validateCaddyImageStorage(image, config string) error {
+	if imageName(image) == officialCaddyImage && CaddyStorageModule(config) == CaddyStorageUncloud {
+		return fmt.Errorf("official Caddy image '%s' doesn't support 'storage uncloud' in the global Caddy config: "+
+			"use an image that includes the Uncloud storage module or a Caddyfile without this option. "+
+			"See %s/concepts/ingress/cluster-storage-for-caddy", image, version.DocsURL)
+	}
+	return nil
+}
+
+// latestCaddyImage returns the latest version of CaddyImage allowed by caddyImageConstraint.
 func latestCaddyImage(ctx context.Context) (reference.NamedTagged, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
@@ -222,7 +361,7 @@ func latestCaddyImage(ctx context.Context) (reference.NamedTagged, error) {
 	return imageWithTag, nil
 }
 
-// latestCaddyTag selects the newest stable 2.x.x tag, falling back to latest.
+// latestCaddyTag selects the newest tag matching caddyImageTagRegex and caddyImageConstraint, falling back to latest.
 func latestCaddyTag(tags []string) string {
 	latestTag := "latest"
 	var latestVersion *semver.Version
@@ -232,7 +371,7 @@ func latestCaddyTag(tags []string) string {
 		}
 
 		v, err := semver.NewVersion(t)
-		if err != nil {
+		if err != nil || !caddyImageConstraint.Check(v) {
 			continue
 		}
 		if latestVersion == nil || v.GreaterThan(latestVersion) {
@@ -243,3 +382,12 @@ func latestCaddyTag(tags []string) string {
 
 	return latestTag
 }
+
+// mustNewConstraint parses a version constraint and panics if it's invalid.
+func mustNewConstraint(constraint string) *semver.Constraints {
+	c, err := semver.NewConstraint(constraint)
+	if err != nil {
+		panic(fmt.Sprintf("invalid version constraint '%s': %v", constraint, err))
+	}
+	return c
+}

pkg/client/caddy_test.go

@@ -5,6 +5,8 @@ import (
 	"testing"
 
 	"github.com/distribution/reference"
+	"github.com/psviderski/uncloud/api/pb"
+	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -15,7 +17,7 @@ func TestLatestCaddyImage(t *testing.T) {
 	image, err := latestCaddyImage(context.Background())
 	require.NoError(t, err)
 
-	assert.Regexp(t, `^caddy:2\.\d+\.\d+$`, reference.FamiliarString(image))
+	assert.Regexp(t, `^ghcr\.io/unlabs-dev/caddy-uncloud:0\.\d+\.\d+$`, reference.FamiliarString(image))
 }
 
 func TestLatestCaddyTag(t *testing.T) {
@@ -27,14 +29,209 @@ func TestLatestCaddyTag(t *testing.T) {
 		want string
 	}{
 		{name: "empty", want: "latest"},
-		{name: "semantic ordering", tags: []string{"2.9.9", "2.11.4", "2.10.0", "2.11.3"}, want: "2.11.4"},
-		{name: "ignore other versions and variants", tags: []string{
-			"latest", "1.0.0", "3.0.0", "2.12.0-rc.1", "2.12.0-alpine", "2.12", "v2.12.0", "2.11.4",
-		}, want: "2.11.4"},
-		{name: "no stable version", tags: []string{"builder", "2.12.0-beta.1"}, want: "latest"},
+		{name: "semantic ordering", tags: []string{"0.1.9", "0.1.11", "0.1.10", "0.1.2"}, want: "0.1.11"},
+		{name: "newest minor version", tags: []string{"0.1.3", "0.2.0", "0.0.9"}, want: "0.2.0"},
+		{name: "ignore newer major versions", tags: []string{"1.0.0", "0.1.3", "2.3.4", "1.0.1"}, want: "0.1.3"},
+		{name: "ignore variants", tags: []string{
+			"latest", "0.1.4-rc.1", "0.1.4-alpine", "0.1", "v0.1.4", "1.0.0-rc.1", "0.1.3",
+		}, want: "0.1.3"},
+		{name: "no stable version", tags: []string{"builder", "0.1.4-beta.1"}, want: "latest"},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			assert.Equal(t, tt.want, latestCaddyTag(tt.tags))
 		})
 	}
 }
+
+func TestDefaultCaddyConfig_UsesUncloudStorage(t *testing.T) {
+	t.Parallel()
+
+	assert.Equal(t, CaddyStorageUncloud, CaddyStorageModule(DefaultCaddyConfig))
+}
+
+func TestCaddyStorageModule(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		name   string
+		config string
+		want   string
+	}{
+		{name: "empty config", want: CaddyStorageFileSystem},
+		{name: "no storage option", config: "{\n\tdebug\n}", want: CaddyStorageFileSystem},
+		{name: "uncloud", config: "{\n\tstorage uncloud\n}", want: CaddyStorageUncloud},
+		{name: "uncloud with options", config: "{\n\tstorage uncloud {\n\t\tfoo bar\n\t}\n}", want: CaddyStorageUncloud},
+		{name: "spaces around", config: "{\n    storage   uncloud  \n}", want: CaddyStorageUncloud},
+		{name: "other module", config: "{\n\tstorage redis\n}", want: "redis"},
+		{name: "explicit file system", config: "{\n\tstorage file_system /data\n}", want: CaddyStorageFileSystem},
+		{name: "commented out", config: "{\n\t# storage uncloud\n}", want: CaddyStorageFileSystem},
+		{name: "similar option", config: "{\n\tstorage_clean_interval 24h\n}", want: CaddyStorageFileSystem},
+		{name: "no module on the same line", config: "{\n\tstorage\n\tdebug\n}", want: CaddyStorageFileSystem},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, CaddyStorageModule(tt.config))
+		})
+	}
+}
+
+func TestIsDefaultCaddyImage(t *testing.T) {
+	t.Parallel()
+
+	assert.True(t, IsDefaultCaddyImage("ghcr.io/unlabs-dev/caddy-uncloud"))
+	assert.True(t, IsDefaultCaddyImage("ghcr.io/unlabs-dev/caddy-uncloud:0.1.3"))
+	assert.False(t, IsDefaultCaddyImage("caddy:2.11.4"))
+	assert.False(t, IsDefaultCaddyImage("example.com/unlabs-dev/caddy-uncloud:0.1.3"))
+	assert.False(t, IsDefaultCaddyImage(""))
+}
+
+func TestIsNewerCaddyImage(t *testing.T) {
+	t.Parallel()
+
+	const repo = "ghcr.io/unlabs-dev/caddy-uncloud"
+	for _, tt := range []struct {
+		name  string
+		image string
+		other string
+		want  bool
+	}{
+		{name: "newer major version", image: repo + ":1.0.0", other: repo + ":0.1.3", want: true},
+		{name: "newer minor version", image: repo + ":0.2.0", other: repo + ":0.1.3", want: true},
+		{name: "same", image: repo + ":0.1.3", other: repo + ":0.1.3"},
+		{name: "older", image: repo + ":0.1.2", other: repo + ":0.1.3"},
+		{name: "different repo", image: "example.com/custom:1.0.0", other: repo + ":0.1.3"},
+		{name: "official image", image: "caddy:2.11.4", other: repo + ":0.1.3"},
+		{name: "tag without version", image: repo + ":latest", other: repo + ":0.1.3"},
+		{name: "no tag", image: repo, other: repo + ":0.1.3"},
+		{name: "fallback to latest tag", image: repo + ":1.0.0", other: repo + ":latest"},
+		{name: "no current image", other: repo + ":0.1.3"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, IsNewerCaddyImage(tt.image, tt.other))
+		})
+	}
+}
+
+func TestCaddyClientNewDeployment(t *testing.T) {
+	t.Parallel()
+
+	const customImage = "example.com/caddy:1.2.3"
+	for _, tt := range []struct {
+		name       string
+		opts       CaddyDeploymentOptions
+		wantConfig string
+		wantLabels map[string]string
+	}{
+		{
+			name:       "defaults",
+			wantConfig: DefaultCaddyConfig,
+			wantLabels: map[string]string{api.LabelCaddyManaged: ""},
+		},
+		{
+			name:       "custom image",
+			opts:       CaddyDeploymentOptions{Image: customImage},
+			wantConfig: DefaultCaddyConfig,
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyImageCustom: ""},
+		},
+		{
+			name:       "custom config",
+			opts:       CaddyDeploymentOptions{Config: new("{\n\tdebug\n}")},
+			wantConfig: "{\n\tdebug\n}",
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
+		},
+		{
+			name:       "custom empty config",
+			opts:       CaddyDeploymentOptions{Config: new("")},
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
+		},
+		{
+			name:       "custom config matching the default is still custom",
+			opts:       CaddyDeploymentOptions{Config: new(DefaultCaddyConfig)},
+			wantConfig: DefaultCaddyConfig,
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
+		},
+		{
+			name: "custom image and config",
+			opts: CaddyDeploymentOptions{
+				Image:  customImage,
+				Config: new("{\n\tdebug\n}"),
+			},
+			wantConfig: "{\n\tdebug\n}",
+			wantLabels: map[string]string{
+				api.LabelCaddyManaged:      "",
+				api.LabelCaddyImageCustom:  "",
+				api.LabelCaddyConfigCustom: "",
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &CaddyClient{client: &Client{}}
+			d, err := c.NewDeployment(context.Background(), tt.opts)
+			require.NoError(t, err)
+
+			assert.Equal(t, tt.wantConfig, d.Spec.CaddyConfig())
+			assert.Equal(t, tt.wantLabels, d.Spec.Labels)
+		})
+	}
+
+	t.Run("official image needs a config without cluster storage", func(t *testing.T) {
+		c := &CaddyClient{client: &Client{}}
+		_, err := c.NewDeployment(context.Background(), CaddyDeploymentOptions{Image: "caddy:2.11.4"})
+		require.ErrorContains(t, err, "doesn't support 'storage uncloud'")
+
+		_, err = c.NewDeployment(context.Background(), CaddyDeploymentOptions{Image: "caddy:2.11.4", Config: new("")})
+		require.NoError(t, err)
+	})
+}
+
+func TestValidateCaddyImageStorage(t *testing.T) {
+	t.Parallel()
+
+	const storageConfig = "{\n\tstorage uncloud\n}"
+	for _, tt := range []struct {
+		name    string
+		image   string
+		config  string
+		wantErr bool
+	}{
+		{name: "official image with uncloud storage", image: "caddy:2.11.4", config: storageConfig, wantErr: true},
+		{name: "official image with registry", image: "docker.io/library/caddy:2", config: storageConfig, wantErr: true},
+		{name: "official image without config", image: "caddy:2.11.4"},
+		{name: "official image with other config", image: "caddy:2.11.4", config: "{\n\tdebug\n}"},
+		{name: "default image with uncloud storage", image: CaddyImage + ":0.1.3", config: storageConfig},
+		{name: "custom image with uncloud storage", image: "example.com/caddy:1", config: storageConfig},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateCaddyImageStorage(tt.image, tt.config)
+			if tt.wantErr {
+				require.ErrorContains(t, err, "doesn't support 'storage uncloud'")
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestOutdatedCaddyDaemons(t *testing.T) {
+	t.Parallel()
+
+	machine := func(name, version string) *pb.MachineMember {
+		return &pb.MachineMember{Machine: &pb.MachineInfo{Name: name, DaemonVersion: version}}
+	}
+
+	assert.Equal(t, []string{"old (0.21.0)", "older (0.20.0)", "unsupported (0.9.3)"},
+		outdatedCaddyDaemons(api.MachineMembersList{
+			machine("old", "0.21.0"),
+			machine("older", "0.20.0"),
+			machine("unsupported", "0.9.3"),
+			machine("no-version", ""),
+			machine("required", "0.22.0"),
+			machine("newer", "0.23.1"),
+			machine("major", "1.0.0"),
+			machine("nightly", "0.22.0-nightly-abc1234"),
+			machine("old-dev-build", "v0.21.0-5-g6c7ddeb-dirty"),
+			machine("local-build", "999.0.0-dev"),
+			machine("not-semver", "6c7ddeb"),
+		}))
+	assert.Empty(t, outdatedCaddyDaemons(api.MachineMembersList{machine("required", "0.22.0")}))
+	assert.Empty(t, outdatedCaddyDaemons(nil))
+}

pkg/client/compose/service.go

@@ -130,6 +130,9 @@ func ServiceSpecFromCompose(project *types.Project, serviceName string) (api.Ser
 		}
 
 		if len(service.Deploy.Labels) > 0 {
+			if err = api.ValidateLabels(service.Deploy.Labels); err != nil {
+				return spec, fmt.Errorf("invalid deploy.labels: %w", err)
+			}
 			spec.Labels = maps.Clone(service.Deploy.Labels)
 		}
 	}

pkg/client/compose/service_test.go

@@ -950,6 +950,42 @@ services:
 	}
 }
 
+func TestServiceSpecFromCompose_InvalidServiceLabels(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		label   string
+		wantErr string
+	}{
+		{name: "reserved prefix", label: api.LabelManaged, wantErr: "reserved prefix"},
+		{name: "reserved daemon prefix", label: "uncloudd.foo", wantErr: "reserved prefix"},
+		{name: "managed service", label: api.LabelReservedPrefix + "service.managed", wantErr: "reserved prefix"},
+		{name: "whitespace in key", label: "my label", wantErr: "must not contain whitespace"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			composeYAML := `
+services:
+  web:
+    image: nginx
+    deploy:
+      labels:
+        - ` + tt.label + `=value
+`
+			project, err := LoadProjectFromContent(context.Background(), composeYAML)
+			require.NoError(t, err)
+
+			_, err = ServiceSpecFromCompose(project, "web")
+			require.ErrorContains(t, err, "invalid deploy.labels")
+			require.ErrorContains(t, err, tt.wantErr)
+		})
+	}
+}
+
 func TestServiceSpecFromCompose_UpdateConfig(t *testing.T) {
 	t.Parallel()
 

pkg/client/deploy/deploy.go

@@ -134,6 +134,22 @@ func (sp *ServicePlan) Format() string {
 		specTable.Row(mod, "image:", formatImageDiff(oldSpec.Container.Image, sp.Spec.Container.Image))
 	}
 
+	// An existing service without an old spec only has new containers added to up-to-date ones so its Caddy config and
+	// labels didn't change.
+	specChanged := sp.IsNewService || oldSpec != nil
+	var old api.ServiceSpec
+	if oldSpec != nil {
+		old = *oldSpec
+	}
+
+	// Caddy config row.
+	if specChanged {
+		mod, value := formatCaddyConfigDiff(old.CaddyConfig(), sp.Spec.CaddyConfig(), sp.IsNewService)
+		if value != "" {
+			specTable.Row(mod, "caddy config:", value)
+		}
+	}
+
 	// Replicas row for replicated services.
 	if sp.Spec.Mode == api.ServiceModeReplicated {
 		replicasStr := fmt.Sprintf("%d", sp.Spec.Replicas)
@@ -153,15 +169,11 @@ func (sp *ServicePlan) Format() string {
 
 	specDiff := specTable.String() + "\n"
 
-	// Render labels as nested blocks outside the table to not widen its attribute column. An existing service without
-	// an old spec only has new containers added to up-to-date ones so its labels didn't change.
-	if sp.IsNewService || oldSpec != nil {
-		var oldLabels, oldContainerLabels map[string]string
-		if oldSpec != nil {
-			oldLabels, oldContainerLabels = oldSpec.Labels, oldSpec.Container.Labels
-		}
-		specDiff += formatLabelsDiff("labels:", oldLabels, sp.Spec.Labels, sp.IsNewService)
-		specDiff += formatLabelsDiff("container labels:", oldContainerLabels, sp.Spec.Container.Labels, sp.IsNewService)
+	// Render the labels outside the table because they span multiple lines.
+	if specChanged {
+		specDiff += formatLabelsDiff("labels:", old.Labels, sp.Spec.Labels, sp.IsNewService)
+		specDiff += formatLabelsDiff("container labels:", old.Container.Labels, sp.Spec.Container.Labels,
+			sp.IsNewService)
 	}
 
 	treePrefix := tui.Faint.Render("  │ ")
@@ -288,6 +300,65 @@ func formatImageDiff(oldImage, newImage string) string {
 		tui.FormatImage(newImage, tui.Green)
 }
 
+// formatCaddyConfigDiff returns the modifier and the number of added and removed lines in the Caddy config. It omits
+// the config content because it can be large. It returns an empty value if the configs are the same.
+func formatCaddyConfigDiff(oldConfig, newConfig string, isNewService bool) (mod, value string) {
+	if oldConfig == newConfig {
+		return "", ""
+	}
+
+	// Count lines regardless of their order, which is enough for a summary and doesn't require a diff algorithm.
+	remaining := make(map[string]int)
+	var removed int
+	if oldConfig != "" {
+		for line := range strings.SplitSeq(oldConfig, "\n") {
+			remaining[line]++
+			removed++
+		}
+	}
+	var added int
+	if newConfig != "" {
+		for line := range strings.SplitSeq(newConfig, "\n") {
+			if remaining[line] > 0 {
+				remaining[line]--
+				removed--
+			} else {
+				added++
+			}
+		}
+	}
+
+	var parts []string
+	if added > 0 {
+		parts = append(parts, tui.Green.Render("+"+strconv.Itoa(added)))
+	}
+	if removed > 0 {
+		parts = append(parts, tui.Red.Render("-"+strconv.Itoa(removed)))
+	}
+
+	switch {
+	case added+removed == 0:
+		value = "changed"
+	case added+removed == 1:
+		value = parts[0] + " line"
+	default:
+		value = strings.Join(parts, " ") + " lines"
+	}
+
+	switch {
+	case isNewService:
+		mod = ""
+	case oldConfig == "":
+		mod = tui.Green.Render("+")
+	case newConfig == "":
+		mod = tui.Red.Render("-")
+	default:
+		mod = "~"
+	}
+
+	return mod, value
+}
+
 // formatLabelsDiff renders a block with the attribute name followed by a nested line for each added, changed, or
 // removed label, sorted by key. It returns an empty string if the labels are the same.
 func formatLabelsDiff(attribute string, oldLabels, newLabels map[string]string, isNewService bool) string {

pkg/client/deploy/deploy_test.go

@@ -83,6 +83,21 @@ func TestServicePlan_Format(t *testing.T) {
 		}
 	}
 
+	withCaddy := func(config string) api.ServiceSpec {
+		s := spec("nginx:1.25", nil, nil)
+		s.Caddy = &api.CaddySpec{Config: config}
+		return s
+	}
+	replaceFrom := func(old api.ServiceSpec) func(api.ServiceSpec) []operation.Operation {
+		return func(new api.ServiceSpec) []operation.Operation {
+			return []operation.Operation{&operation.ReplaceContainerOperation{
+				Spec:         new,
+				MachineName:  "machine-1",
+				OldContainer: newContainer(old),
+			}}
+		}
+	}
+
 	tests := []struct {
 		name         string
 		isNewService bool
@@ -90,6 +105,112 @@ func TestServicePlan_Format(t *testing.T) {
 		ops          func(spec api.ServiceSpec) []operation.Operation
 		expected     []string
 	}{
+		{
+			name: "caddy config lines added and removed",
+			spec: withCaddy("{\n\tstorage uncloud\n}"),
+			ops:  replaceFrom(withCaddy("{\n\tdebug\n\tauto_https off\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ ~ caddy config: +1 -2 lines",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config single line added",
+			spec: withCaddy("{\n\tstorage uncloud\n\tdebug\n}"),
+			ops:  replaceFrom(withCaddy("{\n\tstorage uncloud\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ ~ caddy config: +1 line",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config added to existing service",
+			spec: withCaddy("{\n\tstorage uncloud\n}"),
+			ops:  replaceFrom(spec("nginx:1.25", nil, nil)),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ + caddy config: +3 lines",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config removed",
+			spec: spec("nginx:1.25", nil, nil),
+			ops:  replaceFrom(withCaddy("{\n\tstorage uncloud\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ - caddy config: -3 lines",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config with only reordered lines",
+			spec: withCaddy("a {\n}\nb {\n}"),
+			ops:  replaceFrom(withCaddy("b {\n}\na {\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ ~ caddy config: changed",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config unchanged ignoring surrounding whitespace",
+			spec: withCaddy("{\n\tdebug\n}"),
+			ops:  replaceFrom(withCaddy("\n{\n\tdebug\n}\n")),
+			expected: []string{
+				"~ update service web",
+				"  │   image: nginx:1.25",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name:         "new service with caddy config",
+			isNewService: true,
+			spec:         withCaddy("{\n\tstorage uncloud\n}"),
+			ops: func(new api.ServiceSpec) []operation.Operation {
+				return []operation.Operation{&operation.RunContainerOperation{Spec: new, MachineName: "machine-1"}}
+			},
+			expected: []string{
+				"+ create service web",
+				"  │   image:        nginx:1.25",
+				"  │   caddy config: +3 lines",
+				"  │",
+				"  ╰── +   run container web on machine-1",
+			},
+		},
+		{
+			name:         "caddy config before replicas",
+			isNewService: true,
+			spec: func() api.ServiceSpec {
+				s := withCaddy("{\n\tstorage uncloud\n}")
+				s.Replicas = 3
+				return s
+			}(),
+			ops: func(new api.ServiceSpec) []operation.Operation {
+				return []operation.Operation{&operation.RunContainerOperation{Spec: new, MachineName: "machine-1"}}
+			},
+			expected: []string{
+				"+ create service web",
+				"  │   image:        nginx:1.25",
+				"  │   caddy config: +3 lines",
+				"  │   replicas:     3",
+				"  │",
+				"  ╰── +   run container web on machine-1",
+			},
+		},
 		{
 			name: "service labels added, changed, and removed in place",
 			spec: spec("nginx:1.25", map[string]string{"env": "prod", "team": "platform", "same": "1"}, nil),

test/e2e/service_test.go

@@ -319,6 +319,8 @@ func TestDeployment(t *testing.T) {
 		deployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{})
 		require.NoError(t, err)
 		deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
+		image := deployment.Spec.Container.Image
+		assert.Regexp(t, `^ghcr\.io/unlabs-dev/caddy-uncloud:0\.\d+\.\d+$`, image)
 
 		_, err = deployment.Run(ctx)
 		require.NoError(t, err)
@@ -327,15 +329,72 @@ func TestDeployment(t *testing.T) {
 		require.NoError(t, err)
 		assert.Len(t, svc.Containers, 3)
 		assertServiceMatchesSpec(t, svc, deployment.Spec)
+		assert.Equal(t, map[string]string{api.LabelCaddyManaged: ""}, svc.Spec().Labels)
 
-		ctr := svc.Containers[0].Container
-		assert.Regexp(t, `^caddy:2\.\d+\.\d+$`, ctr.Config.Image)
-
-		config, err := cli.Caddy.Config(ctx, client.CaddyConfigOptions{})
-		require.NoError(t, err)
-
+		var config api.CaddyConfig
+		require.Eventually(t, func() bool {
+			config, err = cli.Caddy.Config(ctx, client.CaddyConfigOptions{})
+			return err == nil && strings.Contains(config.Caddyfile, "storage uncloud")
+		}, 10*time.Second, 200*time.Millisecond, "expected the Caddyfile to enable cluster storage")
+		assert.Empty(t, config.LastReconciliationError)
 		assert.Contains(t, config.Caddyfile, "# Caddyfile autogenerated by Uncloud")
 		assert.Contains(t, config.Caddyfile, "handle /.uncloud-verify")
+
+		// Setting the running image as custom only changes the reserved service labels so it must not recreate
+		// the containers.
+		containerIDs := serviceContainerIDs(svc)
+		deployment, err = cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: image})
+		require.NoError(t, err)
+
+		_, err = deployment.Run(ctx)
+		require.NoError(t, err)
+
+		svc, err = cli.InspectService(ctx, client.CaddyServiceName)
+		require.NoError(t, err)
+		assert.True(t, containerIDs.Equal(serviceContainerIDs(svc)), "expected the containers to be updated in place")
+		assert.Equal(t, map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyImageCustom: ""}, svc.Spec().Labels)
+	})
+
+	t.Run("caddy current spec deployment only adds a container on a new machine", func(t *testing.T) {
+		t.Cleanup(func() {
+			err := cli.RemoveService(ctx, client.CaddyServiceName)
+			if !errors.Is(err, api.ErrNotFound) {
+				require.NoError(t, err)
+			}
+		})
+
+		deployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{})
+		require.NoError(t, err)
+		deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
+
+		_, err = deployment.Run(ctx)
+		require.NoError(t, err)
+
+		svc, err := cli.InspectService(ctx, client.CaddyServiceName)
+		require.NoError(t, err)
+		require.Len(t, svc.Containers, 3)
+
+		// Remove a container to make its machine look like a new one that doesn't run Caddy yet, as 'uc machine add' sees it.
+		removed := svc.Containers[0]
+		err = cli.RemoveContainer(ctx, svc.ID, removed.Container.ID, container.RemoveOptions{Force: true})
+		require.NoError(t, err)
+
+		svc, err = cli.InspectService(ctx, client.CaddyServiceName)
+		require.NoError(t, err)
+		require.Len(t, svc.Containers, 2)
+		containerIDs := serviceContainerIDs(svc)
+
+		_, err = cli.NewDeployment(svc.Spec(), nil).Run(ctx)
+		require.NoError(t, err)
+
+		svc, err = cli.InspectService(ctx, client.CaddyServiceName)
+		require.NoError(t, err)
+		assert.Len(t, svc.Containers, 3)
+		assert.Len(t, serviceMachines(svc).ToSlice(), 3, "expected 1 container on each machine")
+		assert.True(t, containerIDs.IsSubset(serviceContainerIDs(svc)),
+			"expected other containers to be left unchanged")
+		assertServiceMatchesSpec(t, svc, deployment.Spec)
+		assert.Equal(t, map[string]string{api.LabelCaddyManaged: ""}, svc.Spec().Labels)
 	})
 
 	t.Run("caddy with machine placement", func(t *testing.T) {
@@ -365,6 +424,11 @@ func TestDeployment(t *testing.T) {
 		assert.Equal(t, c.Machines[0].ID, svc.Containers[0].MachineID)
 		// initialContainerID := svc.Containers[0].Container.ID
 
+		// Deploying the current spec keeps the placement so it must not add containers on other machines.
+		plan, err := cli.NewDeployment(svc.Spec(), nil).Plan(ctx)
+		require.NoError(t, err)
+		assert.Empty(t, plan.Operations)
+
 		// Deploy to all machines without a placement constraint.
 		deployment, err = cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: image})
 		require.NoError(t, err)
@@ -447,7 +511,7 @@ func TestDeployment(t *testing.T) {
 myapp.example.com {
 	reverse_proxy 1.2.3.4:8000
 }`
-		caddyDeployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Config: caddyCaddyfile})
+		caddyDeployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Config: &caddyCaddyfile})
 		require.NoError(t, err)
 		caddyDeployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
 
@@ -1832,11 +1896,21 @@ func TestServiceLifecycle(t *testing.T) {
 		require.NoError(t, err)
 		assertContainerMatchesSpec(t, mc.Container, spec)
 
-		// Reserved service labels are rejected.
+		// Service labels with an invalid key format are rejected.
 		err = cli.UpdateContainer(ctx, serviceID, ctr.ID, api.UpdateContainerOptions{
-			ServiceLabels: &map[string]string{api.LabelManaged: ""},
+			ServiceLabels: &map[string]string{"my label": ""},
 		})
-		require.ErrorContains(t, err, "reserved prefix")
+		require.ErrorContains(t, err, "must not contain whitespace")
+
+		// The daemon accepts reserved service labels because Uncloud sets its own labels on services.
+		err = cli.UpdateContainer(ctx, serviceID, ctr.ID, api.UpdateContainerOptions{
+			ServiceLabels: &map[string]string{"uncloud.future": ""},
+		})
+		require.NoError(t, err)
+
+		mc, err = cli.InspectContainer(ctx, serviceID, ctr.ID)
+		require.NoError(t, err)
+		assert.Equal(t, map[string]string{"uncloud.future": ""}, mc.Container.ServiceSpec.Labels)
 
 		require.NoError(t, cli.RemoveContainer(ctx, serviceID, ctr.ID, container.RemoveOptions{Force: true}))
 		err = cli.UpdateContainer(ctx, serviceID, ctr.ID, api.UpdateContainerOptions{

website/docs/10-release-notes/_category_.yaml

@@ -0,0 +1,7 @@
+label: Release notes
+collapsed: true # keep the category closed by default
+link:
+  type: generated-index
+  description: What's new in each Uncloud release and how to upgrade.
+# Each release page sets sidebar_position to the negative minor.patch version (for example, -22.0 for v0.22.0)
+# to list the newest release first.

website/docs/10-release-notes/v0.16.0.md

@@ -0,0 +1,96 @@
+---
+sidebar_position: -16.0
+sidebar_label: v0.16.0
+---
+
+# v0.16.0
+
+This page mirrors the [GitHub release](https://github.com/psviderski/uncloud/releases/tag/v0.16.0), which also includes
+the full list of commits.
+
+This is a smaller release focused on reliability improvements, with a new WireGuard network inspection command and expanded Compose spec support for container capabilities and sysctls.
+
+## ✨ Highlights {#highlights}
+
+### WireGuard network inspection
+
+PR: [#161](https://github.com/psviderski/uncloud/issues/161). Thanks to [@jabr](https://github.com/jabr) for the contribution ❤️
+
+The new `uc wg show` command displays the WireGuard network configuration for a machine, including the interface details and all configured peers. This is useful for debugging network connectivity issues and understanding the mesh topology.
+
+Use the `-m/--machine` flag to inspect the configuration of a specific machine in the cluster or the global `--connect user@host` flag to inspect the machine that has issues connecting to the cluster.
+
+```bash
+uc wg show
+uc wg show -m machine-1
+uc wg show --connect user@host
+```
+
+### Kernel capabilities and `sysctls` support
+
+PRs: [#238](https://github.com/psviderski/uncloud/issues/238), [#239](https://github.com/psviderski/uncloud/issues/239). Thanks to [@4ndv](https://github.com/4ndv) for the contribution ❤️
+
+You can now use [`cap_add`](https://github.com/compose-spec/compose-spec/blob/main/spec.md#cap_add), [`cap_drop`](https://github.com/compose-spec/compose-spec/blob/main/spec.md#cap_drop), and [`sysctls`](https://github.com/compose-spec/compose-spec/blob/main/spec.md#sysctls) attribute in your Compose files to add or remove Linux capabilities and configure kernel parameters for service containers. This allows fine-grained control over container privileges and behaviour.
+
+```yaml
+services:
+  app:
+    image: myapp
+    cap_drop:
+      - ALL
+    cap_add:
+      - NET_ADMIN
+    sysctls:
+      net.core.somaxconn: 65535
+      net.ipv4.tcp_syncookies: 1
+```
+
+### Improved cluster initialisation reliability
+
+Machine initialisation and joining with `uc machine init/add` have been made more reliable that fixed WireGuard peer misconfiguration for machines joining the cluster:
+- The daemon now waits for the initial Corrosion state sync and cluster components before serving gRPC requests
+- `uc machine init` and `uc machine add` commands wait for cluster readiness
+- gRPC auto-retries with exponential backoff for transient connection failures (up to ~8 seconds)
+
+:::note
+
+You have to upgrade both the local `uc` CLI and the Uncloud daemon on all your current machines to v0.16.0 to benefit from this improvement.
+
+:::
+
+## Other improvements
+
+- Added `--yes` flag to `uc machine init/add` to auto-confirm machine reset
+- Added `ctx conn` alias for `uc ctx connection` command
+- Reduced gRPC connection timeout to proxied machines from 20s to 10s
+- Machine state file is now saved atomically to prevent corruption
+- Config mounts are now creating non-existent parent directories (#233)
+
+## Upgrade to 0.16.0
+
+### Uncloud CLI locally
+
+To upgrade the Uncloud CLI (`uc`) locally:
+
+```bash
+# Homebrew (macOS, Linux)
+brew upgrade uncloud
+
+# Install script (macOS, Linux)
+curl -fsS https://get.uncloud.run/install.sh | sh
+```
+
+### Machine daemon
+
+To upgrade the Uncloud daemon on your machines, run the following commands on each machine:
+
+```bash
+# AMD64
+curl -fsSL -o uncloudd.tar.gz https://github.com/psviderski/uncloud/releases/download/v0.16.0/uncloudd_linux_amd64.tar.gz
+# ARM64
+# curl -fsSL -o uncloudd.tar.gz https://github.com/psviderski/uncloud/releases/download/v0.16.0/uncloudd_linux_arm64.tar.gz
+tar -xf uncloudd.tar.gz
+sudo install uncloudd /usr/local/bin/uncloudd
+rm uncloudd uncloudd.tar.gz
+sudo systemctl restart uncloud
+```

website/docs/10-release-notes/v0.17.0.md

@@ -0,0 +1,141 @@
+---
+sidebar_position: -17.0
+sidebar_label: v0.17.0
+---
+
+# v0.17.0
+
+This page mirrors the [GitHub release](https://github.com/psviderski/uncloud/releases/tag/v0.17.0), which also includes
+the full list of commits.
+
+This release brings container health monitoring with automatic rollback during rolling deployments, Compose spec support for `healthcheck`, `devices`, and `ulimits`, and various reliability improvements.
+
+## ✨ Highlights {#highlights}
+
+### Container health monitoring and rollback
+
+Changes: [7a9eac5](https://github.com/psviderski/uncloud/commit/7a9eac593f9be0f02b42b1bef5b82f311e6af243), [7b88040](https://github.com/psviderski/uncloud/commit/7b88040c73a43e1dea8db79513fec1c3cd5eac79), [ce3cb8a](https://github.com/psviderski/uncloud/commit/ce3cb8a34f3ae027a4c17b3425eb58c466ce3dbc)
+
+Rolling deployments now monitor crashes and health checks after starting new containers. If a container keeps restarting or becomes unhealthy, the deployment automatically rolls it back and fails to prevent downtime.
+
+https://github.com/user-attachments/assets/4ee73048-9ca2-4c8a-b075-f8c6a0840ee7
+
+#### Compose `healthcheck` support
+
+Compose files now support the [`healthcheck`](https://github.com/compose-spec/compose-spec/blob/main/spec.md#healthcheck) attribute, allowing you to define health checks for your services:
+
+```yaml
+services:
+  app:
+    image: myapp
+    healthcheck:
+      test: curl -f http://localhost:8000/health
+      interval: 5s
+      retries: 3
+      start_period: 10s
+      start_interval: 1s
+```
+
+Use `uc deploy --skip-health` flag to bypass health monitoring for faster emergency deployments.
+
+See [Rolling deployments](/docs/guides/deployments/rolling-deployments) for more details on how rolling deployments work and how to configure health checks for your services.
+
+### Control container replacement order during rolling deployments
+
+PR: [#248](https://github.com/psviderski/uncloud/pull/248). Thanks to [@nick-potts](https://github.com/nick-potts) for the contribution ❤️
+
+You can now control the order of container replacement during rolling deployments with `deploy.update_config.order` in your Compose file:
+- `start-first` (default): starts a new container before stopping the old one
+- `stop-first`: stops the old container before starting a new one (brief downtime)
+
+```yaml
+services:
+  app:
+    image: myapp
+    deploy:
+      update_config:
+        order: stop-first
+```
+
+The default order automatically switches to `stop-first` if a service mounts a volume to prevent data corruption.
+
+See [Rolling deployments](/docs/guides/deployments/rolling-deployments) for more details.
+
+### Compose `devices` support
+
+PR: [#250](https://github.com/psviderski/uncloud/pull/250). Thanks to [@jabr](https://github.com/jabr) for the contribution ❤️
+
+You can now map host devices into containers, for example, Intel integrated GPU for video transcoding using the [`devices`](https://github.com/compose-spec/compose-spec/blob/main/spec.md#devices) attribute in your Compose file:
+
+```yaml
+services:
+  app:
+    image: myapp
+    devices:
+      - /dev/dri:/dev/dri
+      - /dev/ttyUSB0:/dev/ttyUSB0:rw
+```
+
+### Compose `ulimits` support
+
+PR: [#244](https://github.com/psviderski/uncloud/pull/244). Thanks to [@ipaddicting](https://github.com/ipaddicting) for the contribution ❤️
+
+You can now set [`ulimits`](https://github.com/compose-spec/compose-spec/blob/main/spec.md#ulimits) for service containers in your Compose file. For example, to increase the maximum number of open files:
+
+```yaml
+services:
+  app:
+    image: myapp
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536
+```
+
+### Auto-provision volumes for global services
+
+PR: [#243](https://github.com/psviderski/uncloud/pull/243). Thanks to [@zasdaym](https://github.com/zasdaym) for the contribution ❤️
+
+When deploying a [global service](/docs/guides/deployments/deploy-global-services) with a volume, the volume is now automatically created on all machines in the cluster.
+
+## Improvements
+
+- SSH connections with `ssh+cli://` prefix (use `ssh` CLI instead of Go native SSH implementation) reuse a single SSH connection via control socket - significantly speeds up `uc` commands
+- Improved `uc image push` support on macOS by always running proxy to virtualised Docker (except OrbStack that doesn't require it)
+- Global service deployment stops all running conflicting containers, e.g. left from interrupted deploys, not just the first one
+- Temporary socat proxy containers created by `uc image push` are labeled with `uncloud.managed`
+
+## Bug fixes
+
+- Fixed IPv6 address formatting for HTTP URLs ([#254](https://github.com/psviderski/uncloud/pull/254))
+- Fixed `uc image push` when using Rancher Desktop locally ([#251](https://github.com/psviderski/uncloud/pull/251))
+- Fixed `uc machine init` to reset an already initialised machine when using `ssh+cli://` connection
+
+## Upgrade to 0.17.0
+
+### Uncloud CLI locally
+
+To upgrade the Uncloud CLI (`uc`) locally:
+
+```bash
+# Homebrew (macOS, Linux)
+brew upgrade uncloud
+
+# Install script (macOS, Linux)
+curl -fsS https://get.uncloud.run/install.sh | sh
+```
+
+### Machine daemon
+
+To upgrade the Uncloud daemon on your machines, run the following commands on each machine:
+
+```bash
+# AMD64
+curl -fsSL -o uncloudd.tar.gz https://github.com/psviderski/uncloud/releases/download/v0.17.0/uncloudd_linux_amd64.tar.gz
+# ARM64
+# curl -fsSL -o uncloudd.tar.gz https://github.com/psviderski/uncloud/releases/download/v0.17.0/uncloudd_linux_arm64.tar.gz
+tar -xf uncloudd.tar.gz
+sudo install uncloudd /usr/local/bin/uncloudd
+rm uncloudd uncloudd.tar.gz
+sudo systemctl restart uncloud
+```

website/docs/10-release-notes/v0.17.1.md

@@ -0,0 +1,78 @@
+---
+sidebar_position: -17.1
+sidebar_label: v0.17.1
+---
+
+# v0.17.1
+
+This page mirrors the [GitHub release](https://github.com/psviderski/uncloud/releases/tag/v0.17.1), which also includes
+the full list of commits.
+
+This is small patch release that adds WireGuard endpoint management for machines, Compose `stop_grace_period` support, and several networking fixes.
+
+## WireGuard endpoint management
+
+You can now manually specify what WireGuard endpoint to use when adding or initialising a machine instead of relying on auto-detection. This is useful when a machine is behind a NAT or has multiple network interfaces.
+
+```shell
+uc machine init user@host --wg-endpoint 203.0.113.1
+uc machine add user@host --wg-endpoint 203.0.113.2 --wg-endpoint 192.168.1.5
+```
+
+You can also update the WireGuard endpoint for an existing machine:
+
+```shell
+uc machine update my-machine --wg-endpoint 203.0.113.1
+```
+
+## Compose `stop_grace_period` support
+
+Control how long to wait before sending SIGKILL to a container during replacements and stops.
+
+```yaml
+services:
+  app:
+    image: myapp
+    stop_grace_period: 30s
+```
+
+## Improvements
+
+- Tailscale addresses are excluded from auto-detected WireGuard endpoints: see [Discord thread](https://discord.com/channels/1371726032104587335/1477823130670993408)
+- `uc logs` now prints a warning and continues instead of failing when some Compose services are not found in the cluster
+- Improved reliability of the initial cluster store sync by waiting for known missing changes
+- Removed the obsolete `uc machine token` command
+
+## Bug fixes
+
+- Fixed Uncloud DNS resolution for IPv6 machines by creating AAAA records: https://github.com/psviderski/uncloud/pull/257
+- Fixed connection formatting when changing connection with `uc ctx conn`
+
+## Upgrade to 0.17.1
+
+### Uncloud CLI locally
+
+To upgrade the Uncloud CLI (`uc`) locally:
+
+```bash
+# Homebrew (macOS, Linux)
+brew upgrade uncloud
+
+# Install script (macOS, Linux)
+curl -fsS https://get.uncloud.run/install.sh | sh
+```
+
+### Machine daemon
+
+To upgrade the Uncloud daemon on your machines, run the following commands on each machine:
+
+```bash
+# AMD64
+curl -fsSL -o uncloudd.tar.gz https://github.com/psviderski/uncloud/releases/download/v0.17.1/uncloudd_linux_amd64.tar.gz
+# ARM64
+# curl -fsSL -o uncloudd.tar.gz https://github.com/psviderski/uncloud/releases/download/v0.17.1/uncloudd_linux_arm64.tar.gz
+tar -xf uncloudd.tar.gz
+sudo install uncloudd /usr/local/bin/uncloudd
+rm uncloudd uncloudd.tar.gz
+sudo systemctl restart uncloud
+```

website/docs/10-release-notes/v0.18.0.md

@@ -0,0 +1,190 @@
+---
+sidebar_position: -18.0
+sidebar_label: v0.18.0
+---
+
+# v0.18.0
+
+This page mirrors the [GitHub release](https://github.com/psviderski/uncloud/releases/tag/v0.18.0), which also includes
+the full list of commits.
+
+The main highlights of this release are **pre-deploy hooks** for running database migrations and a **redesigned deployment plan** output that makes `uc deploy` much nicer to read and understand.
+
+On top of that, 0.18.0 also:
+- makes the system `ssh` CLI the default connection method (with fallback to the Go implementation)
+- adds Compose support for `pid`, `shm_size`, and the new `x-context` extension to pin a Compose file to a cluster
+- brings new docs, a bunch of quality-of-life improvements and bug fixes, read on for the full details!
+
+## Pre-deploy hooks
+
+⚠️ This requires both `uc` and the daemon to be [upgraded to v0.18.0](#upgrade-to-0180)
+
+Changes: [f4581ea](https://github.com/psviderski/uncloud/commit/f4581eac1167979e9297d749cb8ac8aa56a1cecc), [1c8b770](https://github.com/psviderski/uncloud/commit/1c8b770), [772b31b](https://github.com/psviderski/uncloud/commit/772b31b), [0e820f9](https://github.com/psviderski/uncloud/commit/0e820f9)
+
+You can now run a one-off command before deploying a service using the new `x-pre_deploy` Compose extension. This is ideal for database schema migrations, uploading static assets to a CDN, cache invalidation, and any other setup task that needs to run **once** before new code goes live.
+
+```yaml title="compose.yaml"
+services:
+  web:
+    build: ..
+    environment:
+      DATABASE_URL: postgres://postgres:${DB_PASSWORD}@db:5432/postgres
+    x-pre_deploy:
+      # Apply Django migrations from the built image before deploying new app containers.
+      command: python manage.py migrate
+    depends_on:
+      - db
+```
+
+Here is how it's displayed in the deploy plan when you run `uc deploy`:
+
+<img width="692" height="115" alt="Screenshot 2026-04-10 at 5 16 56 pm" src="https://github.com/user-attachments/assets/fc921bdd-aeea-43e9-923e-abefa2426358" />
+
+The hook command runs in a new container that inherits the service's image, environment variables, volumes, placement, and compute resources.
+
+If the command exits successfully, the deployment continues with a normal rolling update. If it fails or times out, the deployment stops immediately and `uc deploy` displays the latest hook logs to help you diagnose the issue.
+
+![pre-deploy-hook](https://github.com/user-attachments/assets/de05e8db-fb24-4f56-9a1e-4ed28917e65c)
+
+Container tables in `uc ps` and `uc inspect` now also list pre-deploy hook containers.
+
+See the docs:
+- [Pre-deploy hooks](/docs/guides/deployments/pre-deploy-hooks) guide for more details and examples
+- [`x-pre_deploy`](/docs/compose-file-reference/extensions#x-pre_deploy) reference for all available attributes
+
+## Restyled deployment plans
+
+Changes: [0f4c211](https://github.com/psviderski/uncloud/commit/0f4c211), [f5a77b8](https://github.com/psviderski/uncloud/commit/f5a77b8), [b4b1232](https://github.com/psviderski/uncloud/commit/b4b1232)
+
+The deployment plan output that you see when running `uc deploy`, `uc scale`, and `uc caddy deploy` has been redesigned with consistent styling, clearer operation verbs, and a header and the prompt showing the target context.
+
+We hope this makes it easier for you to understand what changes will be made and to which cluster before you confirm the deployment.
+
+<img alt="Screenshot 2026-03-18 at 9 39 38 pm" src="https://github.com/user-attachments/assets/f9d67b36-4a19-4ec6-b115-23c09b90f795" />
+
+## System `ssh` is now the default connection method
+
+Change: [06c2bc5](https://github.com/psviderski/uncloud/commit/06c2bc55f2fbf20920613aac45c07952107160a0)
+
+The `ssh+cli://` connection method that uses your system's `ssh` CLI is now the default for new connections, so the prefix can be omitted:
+
+```shell
+# Old
+uc machine init ssh+cli://user@host
+# New
+uc machine init user@host
+```
+
+`ssh_cli:` can also be replaced with just `ssh:` in the config file (`~/.config/uncloud/config.yaml`):
+
+```yaml title="config.yaml"
+contexts:
+  prod:
+    connections:
+      - ssh: user@host
+```
+
+This will by default use your existing SSH config, agent, keys, `ProxyJump`, and `ProxyCommand` setup automatically. You can fall back to the Go native SSH implementation using `ssh+go://` prefix.
+
+Combined with the SSH control socket reuse introduced in [v0.17.0](https://github.com/psviderski/uncloud/releases/tag/v0.17.0), this makes most `uc` commands noticeably faster and removes a whole class of authentication friction.
+
+The new [Connecting to a cluster](/docs/concepts/clusters/connecting) doc walks through all the supported connection schemes (`ssh://`, `ssh+go://`, `tcp://`, `unix://`), how `uc` picks a machine to talk to from your config, and how to connect without a config file for CI pipelines and scripts.
+
+## Cluster context selection in Compose
+
+Change: [1525d18](https://github.com/psviderski/uncloud/commit/1525d182b516b876510d6e4413ca2fe9ff029b53)
+
+You can now pin a Compose file to a specific context using the new `x-context` extension. This prevents accidentally deploying to the wrong cluster when you switch contexts.
+
+```yaml title="compose.yaml"
+x-context: prod
+
+services:
+  web:
+    image: myapp
+```
+
+When `x-context` is set, `uc deploy` will use that context regardless of your currently selected one in the Uncloud config. The deployment plan output also displays the target context so you always know where your changes are going.
+
+See [`x-context`](/docs/compose-file-reference/extensions#x-context) reference for more details.
+
+## Compose `pid` namespace support
+
+PR: [#276](https://github.com/psviderski/uncloud/pull/276). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+
+You can now configure the [PID namespace mode](https://github.com/compose-spec/compose-spec/blob/main/spec.md#pid) for service containers. The most common use case is `pid: host`, which lets a container see and interact with processes on the host. This is useful for monitoring agents, debuggers, and similar tooling.
+
+```yaml
+services:
+  monitoring:
+    image: agent
+    pid: host
+```
+
+## Compose `shm_size` support
+
+Change: [8ae38cf](https://github.com/psviderski/uncloud/commit/8ae38cfd57d02f42631e0c2bb54c4331b9c072c9)
+
+You can now set [`shm_size`](https://github.com/compose-spec/compose-spec/blob/main/spec.md#shm_size), the size of the `/dev/shm` shared memory tmpfs. This is useful for databases like Postgres, headless Chromium, and other workloads that need a larger shared memory segment.
+
+```yaml
+services:
+  postgres:
+    image: postgres:18
+    shm_size: 256mb
+```
+
+## Documentation
+
+- [Connecting to a cluster](/docs/concepts/clusters/connecting): a complete guide to cluster contexts, connection resolution and all supported types, and how to connect without a config file
+- [CLI configuration file](/docs/cli-config-reference): a full reference for the `~/.config/uncloud/config.yaml` file, covering contexts, connections, and all supported fields
+- [Pre-deploy hooks](/docs/guides/deployments/pre-deploy-hooks): a guide for the new feature, with examples for database migrations and other one-off setup tasks
+- [Compose extensions](/docs/compose-file-reference/extensions): a dedicated reference page for all Uncloud-specific Compose extensions
+
+## Improvements
+
+- The `uc` CLI and the daemon now exchange versions on every gRPC call and error if they are incompatible. This is a groundwork to make better user experience when releasing breaking changes in the future ([#260](https://github.com/psviderski/uncloud/pull/260)). Thanks to [@jabr](https://github.com/jabr) for the contribution ❤️
+- New `uc exec` accepts an optional `--` separator before the command, matching Docker and other CLIs ([#301](https://github.com/psviderski/uncloud/pull/301)). Thanks to [@nick-potts](https://github.com/nick-potts) for the contribution ❤️
+- New `uc docs --manual` flag generates manual pages for `uc`, making it easier for distribution maintainers to package manpages alongside the binary ([#302](https://github.com/psviderski/uncloud/pull/302)). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+- The daemon now exposes a gRPC method to fetch journald logs from a machine, laying the groundwork for an upcoming `uc machine logs` command ([#282](https://github.com/psviderski/uncloud/pull/282)). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+- Warnings are now printed for detected unsupported Compose features ([#288](https://github.com/psviderski/uncloud/pull/288)). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+- New `--auto-confirm` flag for `uc scale` which now always asks for confirmation before changing replica counts
+- New `--ulimit` flag for `uc run` to set resource limits
+- bumped Go to 1.26.1 and modernized the codebase using Go 1.26 ([#278](https://github.com/psviderski/uncloud/pull/278))
+- Migrated TUI components to `lipgloss/v2`, `bubbletea/v2`, and `huh/v2`
+
+## Bug fixes
+
+- Fixed `uc deploy` crash in an initialised but empty git repository ([#275](https://github.com/psviderski/uncloud/pull/275)). Thanks to [@tonyo](https://github.com/tonyo) for the contribution ❤️
+- Fixed `ucind` downloading the wrong Corrosion architecture on arm64 ([#285](https://github.com/psviderski/uncloud/pull/285)). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+- Fixed output contamination in the SSH CLI executor by separating stdout and stderr ([#270](https://github.com/psviderski/uncloud/issues/270))
+- Fixed trying other machine connections when the first one fails for the new default `ssh` type. Fixed the TUI progress spinner.
+
+## Upgrade to 0.18.0
+
+### Uncloud CLI locally
+
+To upgrade the Uncloud CLI (`uc`) locally:
+
+```bash
+# Homebrew (macOS, Linux)
+brew upgrade uncloud
+
+# Install script (macOS, Linux)
+curl -fsS https://get.uncloud.run/install.sh | sh
+```
+
+### Machine daemon
+
+To upgrade the Uncloud daemon on your machines, run the following commands on each machine:
+
+```bash
+# AMD64
+curl -fsSL -o uncloudd.tar.gz https://github.com/psviderski/uncloud/releases/download/v0.18.0/uncloudd_linux_amd64.tar.gz
+# ARM64
+# curl -fsSL -o uncloudd.tar.gz https://github.com/psviderski/uncloud/releases/download/v0.18.0/uncloudd_linux_arm64.tar.gz
+tar -xf uncloudd.tar.gz
+sudo install uncloudd /usr/local/bin/uncloudd
+rm uncloudd uncloudd.tar.gz
+sudo systemctl restart uncloud
+```

website/docs/10-release-notes/v0.19.0.md

@@ -0,0 +1,170 @@
+---
+sidebar_position: -19.0
+sidebar_label: v0.19.0
+---
+
+# v0.19.0
+
+This page mirrors the [GitHub release](https://github.com/psviderski/uncloud/releases/tag/v0.19.0), which also includes
+the full list of commits.
+
+This release brings **easier debugging** of failed deploys, new **troubleshooting commands** (`uc machine logs` and `uc machine rtt`), and a **nightly release** channel.
+
+Read on for the full list of changes and upgrade instructions.
+
+## Show logs from a failed deploy
+
+Changes: [8afe523](https://github.com/psviderski/uncloud/commit/8afe523), [303c8e4](https://github.com/psviderski/uncloud/commit/303c8e4), [56a54ab](https://github.com/psviderski/uncloud/commit/56a54ab)
+
+When `uc deploy` fails because a pre-deploy hook failed or a new container couldn't become healthy, it now prints the last 10 (configurable) log lines from the failed container.
+
+This is a quality-of-life improvement that saves you from running a follow-up `uc logs` to figure out what went wrong.
+
+<img width="1253" height="871" alt="Screenshot 2026-04-24 at 2 01 22 pm" src="https://github.com/user-attachments/assets/69adcf19-ebe8-465e-8834-414c99c15260" />
+
+See [Failed container logs](/docs/guides/deployments/rolling-deployments/#failed-container-logs) for more details.
+
+## Filter service logs by container
+
+Change: [2045819](https://github.com/psviderski/uncloud/commit/2045819)
+
+`uc logs` now accepts a `SERVICE/CONTAINER` form where `CONTAINER` is a container name, full ID, or unique ID prefix. This is handy when one replica of a service is misbehaving and you want to look at it without the noise from the others:
+
+```shell
+uc logs web/2f60
+# Mix and match
+uc logs api/61d57fd3428f web/2f60 db
+```
+
+## Stream machine logs
+
+⚠️ This requires both `uc` and the daemon to be [upgraded to v0.19.0](#upgrade-to-0190)
+
+PRs: [#282](https://github.com/psviderski/uncloud/pull/282), [#283](https://github.com/psviderski/uncloud/pull/283). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+
+You can now stream logs from the systemd services that run Uncloud itself on remote machines using the new `uc machine logs` command. It covers three main services:
+
+- `uncloud` - the Uncloud daemon
+- `uncloud-corrosion` - the Corrosion service providing the distributed cluster store
+- `docker` - the Docker daemon
+
+For example, stream the Uncloud daemon logs from all machines with:
+```shell
+uc machine logs -f uncloud
+```
+
+This is useful for troubleshooting Uncloud operations without having to SSH into every machine and run `journalctl` yourself.
+
+See [`uc machine logs`](/docs/cli-reference/uc_machine_logs) for more details and examples.
+
+## Round-trip time between machines
+
+⚠️ This requires both `uc` and the daemon to be [upgraded to v0.19.0](#upgrade-to-0190)
+
+PR: [#226](https://github.com/psviderski/uncloud/pull/226). Thanks to [@jabr](https://github.com/jabr) for the contribution ❤️
+
+The new `uc machine rtt` command shows the round-trip time between every pair of machines in the cluster. This gives you a real-time view of how the mesh network is performing without needing to run manual `ping` tests.
+
+The data is collected from Corrosion's gossip protocol, which samples latency between peers as part of its normal operation.
+
+```shell
+$ uc machine rtt
+MACHINE     PEER        MEDIAN   STDDEV
+machine-1   machine-2   140ms    ±19.4ms
+machine-1   machine-3   39ms     ±1.1ms
+machine-2   machine-1   168ms    ±18.5ms
+machine-2   machine-3   203ms    ±42.3ms
+machine-3   machine-1   40ms     ±2.0ms
+machine-3   machine-2   158ms    ±15.2ms
+```
+
+A new `RTT` column has also been added to `uc wg show` to show the median round-trip time to each WireGuard peer.
+
+## Local machine upstreams first in Caddyfile
+
+Change: [45cf87a](https://github.com/psviderski/uncloud/commit/45cf87a)
+
+The generated Caddyfile now lists upstreams from the local machine first for each service.
+
+On its own this doesn't change routing behaviour (Caddy's default `random` load balancing policy ignores order). But if you pair it with the `first` policy in a custom Caddy config, you can always prefer the same-host replica and only fall back to remote machines when the local one is unhealthy.
+
+```yaml
+services:
+  app:
+    ...
+    x-caddy: |
+      example.com {
+          reverse_proxy {{upstreams 8000}} {
+              import common_proxy
+              lb_policy first
+          }
+          log
+      }
+```
+
+This saves a cross-machine WireGuard hop for every request that hits Caddy on the machine which already has a replica of the target service. This is especially useful if you have a multi-region setup.
+
+## Nightly builds
+
+PR: [#308](https://github.com/psviderski/uncloud/pull/308). Thanks to [@tonyo](https://github.com/tonyo) for the contribution ❤️
+
+Every push to `main` now produces a set of nightly binaries tagged as the [`nightly` release](https://github.com/psviderski/uncloud/releases/tag/nightly) on GitHub.
+
+They're great for testing upcoming unreleased changes and reporting regressions early. They might be unstable, so please don't run them in production.
+
+Install the nightly `uc` CLI locally:
+
+```shell
+curl -fsS https://get.uncloud.run/install.sh | VERSION=nightly sh
+```
+
+Initialise a cluster or add a machine with a nightly daemon:
+
+```shell
+uc machine init --version nightly user@host
+uc machine add --version nightly user@host
+```
+
+## Improvements
+
+- `uc machine init/add` now embed the install script into the `uc` binary and send it to the remote machine over the existing SSH connection instead of `curl | bash` ([fe829ef](https://github.com/psviderski/uncloud/commit/fe829ef)).
+- New `uc ctx show` command prints the name of the currently active cluster context, great for shell prompts and scripts ([#317](https://github.com/psviderski/uncloud/pull/317)).
+- SSH connections now use `-o StrictHostKeyChecking=accept-new`, which silently accepts host keys on first connection but still protects against key changes later ([#303](https://github.com/psviderski/uncloud/pull/303)).
+- The `STORE` column in `uc images` is hidden when all machines use the containerd image store, which is the default for new clusters ([c38d916](https://github.com/psviderski/uncloud/commit/c38d916))
+- Image push errors now include the underlying error from the unregistry proxy, which makes it much easier to tell apart a broken image push from a broken SSH tunnel ([d340659](https://github.com/psviderski/uncloud/commit/d340659))
+- `ucind` now accepts a `UNCLOUD_CONFIG` environment variable to override the config file path for development clusters ([#315](https://github.com/psviderski/uncloud/pull/315)).
+- New `ucind cluster ls` command lists the local development clusters managed by `ucind` with their machines ([#316](https://github.com/psviderski/uncloud/pull/316)).
+
+## Bug fixes
+
+- Fixed containerd socket auto-detection for unregistry after a machine reboot ([47199c3](https://github.com/psviderski/uncloud/commit/47199c3)).
+- Fixed Caddy config being regenerated on every service change because container records were serialised non-deterministically ([#111](https://github.com/psviderski/uncloud/issues/111)).
+- Fixed SSH control socket path on WSL2 when the runtime directory doesn't exist ([#319](https://github.com/psviderski/uncloud/issues/319)).
+- Fixed a stale SSH `ControlMaster` connection sometimes causing `uc machine init`/`add` to hang ([fdbffbf](https://github.com/psviderski/uncloud/commit/fdbffbf)).
+
+## Upgrade to 0.19.0
+
+### Uncloud CLI locally
+
+To upgrade the Uncloud CLI (`uc`) locally:
+
+```bash
+# Homebrew (macOS, Linux)
+brew upgrade uncloud
+
+# Install script (macOS, Linux)
+curl -fsS https://get.uncloud.run/install.sh | sh
+```
+
+### Machine daemon
+
+To upgrade the Uncloud daemon on your machines, run the following commands on each machine:
+
+```bash
+ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')
+curl -fsSL -o uncloudd.tar.gz https://github.com/psviderski/uncloud/releases/download/v0.19.0/uncloudd_linux_${ARCH}.tar.gz
+tar -xf uncloudd.tar.gz
+sudo install uncloudd /usr/local/bin/uncloudd
+rm uncloudd uncloudd.tar.gz
+sudo systemctl restart uncloud
+```

website/docs/10-release-notes/v0.20.0.md

@@ -0,0 +1,146 @@
+---
+sidebar_position: -20.0
+sidebar_label: v0.20.0
+---
+
+# v0.20.0
+
+This page mirrors the [GitHub release](https://github.com/psviderski/uncloud/releases/tag/v0.20.0), which also includes
+the full list of commits.
+
+This release adds **secrets** for passing sensitive data to services, a new **`uc proxy`** command for port-forwarding, a **Prometheus metrics** endpoint, **`uc machine ls`** that surfaces each machine's OS, Docker, and daemon versions. Plus daemon self-upgrade on `uc machine init/add`, WireGuard MTU auto-detection, shell completion, and more.
+
+⚠️ **This is a breaking release.** Both the `uc` CLI and the `uncloudd` daemon on **every** machine must be upgraded to 0.20. See [Breaking changes](#breaking-changes) below before you upgrade.
+
+## ⚠️ Breaking changes {#breaking-changes}
+
+0.20 changes the internal gRPC API proxy format and migrates the cluster store to a new Corrosion version, so a mixed 0.19/0.20 cluster will not work. The built-in version check rejects requests between incompatible CLI and daemon versions, so upgrade everything together. See [Upgrade to 0.20.0](#upgrade-to-0200) for instructions.
+
+- **Server-side gRPC proxy resolution** ([#247](https://github.com/psviderski/uncloud/pull/247)). API proxy targets (machines) are now resolved on the daemon instead of the CLI. This changes the request format on the wire, so old and new components can't talk to each other. Thanks to [@jabr](https://github.com/jabr) for the contribution ❤️
+- **Corrosion now runs as a managed container** ([71da325](https://github.com/psviderski/uncloud/commit/71da325)). The Corrosion cluster store moved from a systemd unit to a Docker container and was upgraded to Corrosion v1 ([`2026.6.15`](https://github.com/unlabs-dev/corrosion/releases/tag/v2026.6.15)). Existing data is migrated to the new format **automatically** on first daemon start. Auth is now enabled for local Corrosion API requests too ([#110](https://github.com/psviderski/uncloud/issues/110)).
+- **CLI artifact renamed `uncloud_*` → `uc_*`** ([a5d73fe](https://github.com/psviderski/uncloud/commit/a5d73fe)). Homebrew and the install script handle this for you. Only update your own scripts if you download the CLI archive directly. The daemon archive name (`uncloudd_*`) is unchanged.
+- **Unregistry image proxy moved from port `5000` to `51500`** ([d36ff67](https://github.com/psviderski/uncloud/commit/d36ff67)). `uc` handles this automatically when pushing images. Only update your setup if you reach this proxy port directly in your own tooling.
+
+## Secrets support
+
+PR: [#403](https://github.com/psviderski/uncloud/pull/403), docs: [e01c8d3](https://github.com/psviderski/uncloud/commit/e01c8d3)
+
+You can now keep sensitive data like passwords, API tokens, and private keys out of your Compose file. Define a secret with the new `x-command` extension and reference it from a service's `environment` with `secret://<name>`. The command runs locally at deploy time and pulls the value from your password manager, cloud secrets store, or any CLI you already use.
+
+```yaml title="compose.yaml"
+services:
+  api:
+    image: myapp:latest
+    environment:
+      DB_PASSWORD: secret://db_password
+
+secrets:
+  db_password:
+    x-command: op read "op://prod/myapp/db_password"
+```
+
+The value never lands in your Compose file or git. See the new [Secrets](/docs/concepts/secrets) doc for supported sources, caveats, and examples.
+
+## Proxy a service port locally
+
+PR: [#384](https://github.com/psviderski/uncloud/pull/384). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+
+The new `uc proxy` command forwards a service port to a local port over the cluster network, similar to `kubectl port-forward`. This is handy for reaching an internal database or admin UI from your laptop without exposing them publicly.
+
+## Richer `uc machine ls`
+
+Changes: [8aa0a96](https://github.com/psviderski/uncloud/commit/8aa0a96), [ad43569](https://github.com/psviderski/uncloud/commit/ad43569), [1d5c717](https://github.com/psviderski/uncloud/commit/1d5c717), [23d6b44](https://github.com/psviderski/uncloud/commit/23d6b44)
+
+`uc machine ls` now shows the `OS`, `KERNEL`, `ARCH`, `DOCKER`, and `VERSION` (daemon) of each machine. You can spot version drift and outdated daemons at a glance.
+
+```shell
+$ uc machine ls
+NAME        STATE   ADDRESS         PUBLIC IP      WIREGUARD ENDPOINTS   OS                   KERNEL              ARCH    DOCKER   VERSION
+machine-1   Up      10.210.0.1/24   203.0.113.10   203.0.113.10:51820    Debian 12.12         6.1.0-41-amd64      amd64   29.4.0   0.20.0
+machine-2   Up      10.210.1.1/24   198.51.100.7   198.51.100.7:51820    Debian 12.5          6.1.0-18-amd64      amd64   26.0.0   0.20.0
+machine-3   Up      10.210.2.1/24   -              192.0.2.44:51820      Ubuntu 22.04.5 LTS   6.8.0-1050-oracle   arm64   29.5.3   0.20.0
+```
+
+Pass `-o json` for machine-readable output to use in scripts.
+
+## Prometheus metrics
+
+PR: [#304](https://github.com/psviderski/uncloud/pull/304). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+
+Each daemon now serves the initial set of Prometheus metrics on `http://<machine-ip>:51090/metrics`, listening on the machine's internal IP. Point your scraper at it to monitor the cluster.
+
+## Daemon self-upgrade on machine init/add
+
+Change: [a26af3b](https://github.com/psviderski/uncloud/commit/a26af3b)
+
+`uc machine init` and `uc machine add` now upgrade the installed daemon binary to the latest (or a specified `--version`) if the daemon was already installed on the machine.
+
+## Improvements
+
+- WireGuard now auto-detects the optimal interface MTU, with a new `--wg-mtu` flag to override it ([0c3b8b1](https://github.com/psviderski/uncloud/commit/0c3b8b1)).
+- New `--wg-port` flag for `uc machine init/add` to customise the WireGuard listen port ([#366](https://github.com/psviderski/uncloud/pull/366)). Thanks to [@dasunsrule32](https://github.com/dasunsrule32) for the contribution ❤️
+- `x-ports` can bind to multiple host IP addresses given as a CIDR prefix ([#358](https://github.com/psviderski/uncloud/pull/358)). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+- New `uc version` and `uncloudd version` commands that print full build information, replacing the `--version` flag ([2f15915](https://github.com/psviderski/uncloud/commit/2f15915)).
+- `uc machine init/add` installs `curl` on the remote machine if it's missing ([#402](https://github.com/psviderski/uncloud/issues/402)).
+- New `uc caddy logs` alias for `uc logs caddy` ([#381](https://github.com/psviderski/uncloud/pull/381)).
+- Shell/argument completion for the `uc` and `ucind` CLIs (machines, services, volumes) ([#309](https://github.com/psviderski/uncloud/pull/309), [#344](https://github.com/psviderski/uncloud/pull/344), [#352](https://github.com/psviderski/uncloud/pull/352)). Thanks to [@miekg](https://github.com/miekg) for the contributions ❤️
+- Support rootless Docker locally when pushing images to the cluster ([#347](https://github.com/psviderski/uncloud/pull/347)). Thanks to [@tonyo](https://github.com/tonyo) for the contribution ❤️
+- The CLI strips ANSI colors from table output and skips interactive TUI elements when stdout is not a TTY ([e3823c2](https://github.com/psviderski/uncloud/commit/e3823c2), [#386](https://github.com/psviderski/uncloud/issues/386)).
+
+## Bug fixes
+
+- Reject relative volume source paths in Compose with a clear error ([#353](https://github.com/psviderski/uncloud/pull/353)). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+- Return a non-zero exit code when a `uc` command is cancelled by declining a confirmation prompt ([#354](https://github.com/psviderski/uncloud/issues/354)).
+- Don't include outdated containers from a removed machine in the DNS and Caddy configs ([07acb31](https://github.com/psviderski/uncloud/commit/07acb31)).
+- Improve context cancellation and Corrosion resubscription handling during watch and sync operations ([328d175](https://github.com/psviderski/uncloud/commit/328d175), [83256b5](https://github.com/psviderski/uncloud/commit/83256b5), [99e1ae0](https://github.com/psviderski/uncloud/commit/99e1ae0), [5f07bc5](https://github.com/psviderski/uncloud/commit/5f07bc5)).
+- Use a forked Bubble Tea that drains pending TTY input on shutdown ([#286](https://github.com/psviderski/uncloud/pull/286)).
+
+## Upgrade to 0.20.0
+
+Upgrade **all** machines and your local CLI to 0.20. Corrosion data migrates automatically on the first daemon restart.
+
+:::important
+
+On the first restart, each daemon pulls the Corrosion container image `ghcr.io/unlabs-dev/corrosion:2026.6.15`. Make sure your machines can reach `ghcr.io`, or pre-pull the image, otherwise the cluster store won't start.
+
+:::
+
+### Uncloud CLI locally
+
+```bash
+# Homebrew (macOS, Linux)
+brew upgrade uncloud
+
+# Install script (macOS, Linux)
+curl -fsS https://get.uncloud.run/install.sh | sh
+```
+
+### Machine daemon
+
+Run the following commands on each machine:
+
+```bash
+ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')
+curl -fsSL -o uncloudd.tar.gz https://github.com/psviderski/uncloud/releases/download/v0.20.0/uncloudd_linux_${ARCH}.tar.gz
+tar -xf uncloudd.tar.gz
+sudo install uncloudd /usr/local/bin/uncloudd
+rm uncloudd uncloudd.tar.gz
+sudo systemctl restart uncloud
+```
+
+### After upgrading
+
+On the first restart, the daemon migrates the Corrosion store and stops the legacy systemd service. A couple of leftovers need manual cleanup on each machine:
+
+- The old store is preserved at `/var/lib/uncloud/corrosion.backup-<timestamp>`. Once you've verified the cluster is healthy (`uc machine ls`), you can remove it:
+
+  ```bash
+  sudo rm -rf /var/lib/uncloud/corrosion.backup-*
+  ```
+
+- The legacy Corrosion systemd unit and binary are left in place because the daemon's sandbox doesn't have permission to delete them. Remove them manually:
+
+  ```bash
+  sudo rm /etc/systemd/system/uncloud-corrosion.service /usr/local/bin/uncloud-corrosion
+  sudo systemctl daemon-reload
+  ```

website/docs/10-release-notes/v0.21.0.md

@@ -0,0 +1,184 @@
+---
+sidebar_position: -21.0
+sidebar_label: v0.21.0
+---
+
+# v0.21.0
+
+This page mirrors the [GitHub release](https://github.com/psviderski/uncloud/releases/tag/v0.21.0), which also includes
+the full list of commits.
+
+This release brings **shared TLS certificate storage for Caddy**, **systemd socket activation**, **runtime templates**, and a way to use **your own cluster domain**. It also adds a certificate inventory command, faster service listing, and fixes for proxying, container logs, and terminal output.
+
+## ⚠️ Breaking changes {#breaking-changes}
+
+- The API socket moved from `/run/uncloud/uncloud.sock` to `/run/uncloud/api/uncloud.sock` ([f68c985](https://github.com/psviderski/uncloud/commit/f68c985)). Update your custom tools and container mounts if you used the API socket directly. Both old and new `uc` versions handle the socket move automatically.
+
+## Shared certificate storage for Caddy
+
+Caddy module: [caddy-uncloud](https://github.com/unlabs-dev/caddy-uncloud), issue: [#31](https://github.com/psviderski/uncloud/issues/31)
+
+If you run Caddy on multiple machines behind a load balancer or a DNS record with multiple IPs, certificate issuance may not work correctly.
+
+The new Uncloud storage module lets Caddy instances share certificates, private keys, and challenge tokens through the cluster store. Any instance can read the token to answer a challenge, and once one obtains a certificate, the others can use it too. Distributed locks coordinate issuance across machines.
+
+Shared storage is **opt-in** for now. See [Cluster storage for Caddy](/docs/concepts/ingress/cluster-storage-for-caddy) for more details.
+
+## Inspect Caddy certificates
+
+Changes: [930e7ea](https://github.com/psviderski/uncloud/commit/930e7ea), [9c21ae6](https://github.com/psviderski/uncloud/commit/9c21ae6)
+
+The new [`uc caddy cert ls`](/docs/cli-reference/uc_caddy_cert_ls) command lists certificates in Caddy's shared cluster storage. This is handy for checking which certificates have been issued and when they expire.
+
+```shell
+uc caddy cert ls
+
+ID             NAME                            ISSUER                    EXPIRES
+c111e23615f7   dns.uncloud.run                 Let's Encrypt             2026-12-31 (2 months)
+b5dcd2a03e1b   nginx.2t5ex2.uncld.dev          Let's Encrypt             2026-12-31 (2 months)
+70707fd2fea2   test-staging.2t5ex2.uncld.dev   Let's Encrypt (staging)   2026-12-31 (2 months)
+c5f9301e1c06   uncloud.run                     Let's Encrypt             2026-12-31 (2 months)
+```
+
+```shell
+uc caddy cert ls --machine machine-1
+uc caddy cert ls -o json
+```
+
+This requires Caddy to first be deployed with [cluster storage](/docs/concepts/ingress/cluster-storage-for-caddy).
+
+## Systemd socket activation
+
+Change: [f68c985](https://github.com/psviderski/uncloud/commit/f68c985)
+
+Systemd now creates the Uncloud API socket and passes it to the daemon when it starts. The socket is available before Docker starts and stays in place across daemon restarts. This gives Caddy's cluster storage module and other extensions a stable API socket, including after a reboot.
+
+## Runtime templates for bind mounts
+
+Change: [5c33e4b](https://github.com/psviderski/uncloud/commit/5c33e4b)
+
+You can now use per-container metadata in bind mount paths. Uncloud renders these Go templates on the destination machine just before creating each container, so every replica can get its own host directory:
+
+```yaml
+services:
+  app:
+    image: app:latest
+    scale: 2
+    volumes:
+      - "/var/lib/app/{{.Container.Name}}:/data"
+```
+
+For now, runtime templates expose `.Container.Name` and work only in the host and container paths of bind mounts. They could easily be extended to other metadata fields and Compose attributes. Please share your use case on [#431](https://github.com/psviderski/uncloud/issues/431).
+
+See [Runtime templates](/docs/concepts/services/runtime-templates) for more details and examples.
+
+## Use your own cluster domain
+
+PR: [#365](https://github.com/psviderski/uncloud/pull/365). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+
+The new [`uc dns set`](/docs/cli-reference/uc_dns_set) command lets you use a domain you manage yourself as the default cluster domain for generated service hostnames:
+
+```shell
+uc dns set apps.example.com
+```
+
+Configure wildcard DNS records for `*.apps.example.com` with your DNS provider, pointing to machines running Caddy. New services published without an explicit hostname can then use addresses like `web.apps.example.com`. Uncloud doesn't create or manage those external records.
+
+If your cluster already has a reserved `*.uncld.dev` domain, release that reservation with `uc dns release` first.
+
+## Improvements
+
+- `uc ls` now collects services and containers with a single broadcast request instead of inspecting every service separately. This significantly speeds it up, especially when you have many services ([0e53f50](https://github.com/psviderski/uncloud/commit/0e53f50)).
+- Compose now supports `stdin_open` and `tty` ([#419](https://github.com/psviderski/uncloud/pull/419)). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+- Compose warns about more unsupported options, including external configs, `container_name`, and `deploy.placement` ([#429](https://github.com/psviderski/uncloud/pull/429)). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+- `uc machine init/add` now accept `UNCLOUD_DAEMON_VERSION` as an alternative to `--version`, useful for scripts and nightly setups ([#409](https://github.com/psviderski/uncloud/pull/409)). Thanks to [@tonyo](https://github.com/tonyo) for the contribution ❤️
+- Caddy deployments now have a healthcheck that queries the admin API, so the deployment waits for Caddy to load its config before treating a new container as healthy ([70de7b1](https://github.com/psviderski/uncloud/commit/70de7b1)).
+- The [installation docs](/docs/getting-started/install-cli) now include mise ([#421](https://github.com/psviderski/uncloud/pull/421)).
+
+## Bug fixes
+
+- Preserve custom global Caddy configuration during regeneration on Caddy restart. `uc caddy config` now warns when the last attempt to load the config failed ([ace8cbf](https://github.com/psviderski/uncloud/commit/ace8cbf)).
+- Keep `uc proxy` running when an individual client disconnects or a forwarded connection fails. Connection errors now include more useful troubleshooting information ([fa77edf](https://github.com/psviderski/uncloud/commit/fa77edf)).
+- Stream logs correctly from containers with an allocated TTY ([#419](https://github.com/psviderski/uncloud/pull/419)).
+- Stop terminal capability replies from leaking into the shell after CLI spinners finish ([#435](https://github.com/psviderski/uncloud/pull/435)). Thanks to [@tonyo](https://github.com/tonyo) for the contribution ❤️
+- Fix shell completion when using `--connect`, `--context`, or `--uncloud-config` ([351698c](https://github.com/psviderski/uncloud/commit/351698c)).
+- Return a non-zero exit code consistently when a command is cancelled by declining a confirmation prompt ([#406](https://github.com/psviderski/uncloud/pull/406)). Thanks to [@miekg](https://github.com/miekg) for the contribution ❤️
+- Extend the systemd startup timeout while pulling the Corrosion image, allowing up to 5 minutes for a slow pull ([dc721e5](https://github.com/psviderski/uncloud/commit/dc721e5)).
+- Fix daemon version comparisons when upgrading from a nightly build to the latest stable release ([50f8fbc](https://github.com/psviderski/uncloud/commit/50f8fbc)).
+- Avoid waiting on replication gaps from unavailable cluster members after the requested store versions have been reached ([e4bd1ad](https://github.com/psviderski/uncloud/commit/e4bd1ad)).
+
+## Upgrade to v0.21.0
+
+Upgrade your local CLI and the daemon on every machine. If you're upgrading from before v0.20.0, follow the [v0.20.0 upgrade instructions](https://github.com/psviderski/uncloud/releases/tag/v0.20.0#upgrade-to-0200) first for the Corrosion migration.
+
+### Uncloud CLI locally
+
+```bash
+# Homebrew (macOS, Linux)
+brew upgrade uncloud
+
+# Install script (macOS, Linux)
+curl -fsS https://get.uncloud.run/install.sh | VERSION=0.21.0 sh
+```
+
+### Machine daemon
+
+Run the following commands over SSH on each machine to upgrade the daemon binary:
+
+```bash
+ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')
+curl -fsSL -o uncloudd.tar.gz https://github.com/psviderski/uncloud/releases/download/v0.21.0/uncloudd_linux_${ARCH}.tar.gz
+tar -xf uncloudd.tar.gz
+sudo install uncloudd /usr/local/bin/uncloudd
+rm uncloudd uncloudd.tar.gz
+```
+
+**The following systemd unit updates are only needed on existing machines.** New machines set up with `uc machine init` or `uc machine add` using v0.21.0 or later install these units automatically. Replacing the daemon binary alone on an existing machine doesn't install the new `uncloud.socket` unit or update `uncloud.service`.
+
+Create `/etc/systemd/system/uncloud.socket`:
+
+```bash
+sudo tee /etc/systemd/system/uncloud.socket >/dev/null <<'EOF'
+[Unit]
+Description=Uncloud API socket
+Before=docker.service
+
+[Socket]
+ListenStream=/run/uncloud/api/uncloud.sock
+SocketUser=root
+SocketGroup=uncloud
+SocketMode=0660
+DirectoryMode=0750
+
+[Install]
+WantedBy=sockets.target
+EOF
+```
+
+Run this command to add the socket dependency directly to `/etc/systemd/system/uncloud.service`:
+
+```bash
+sudo sed -i \
+  's/^After=.*$/Requires=uncloud.socket\
+After=network-online.target uncloud.socket docker.service/' \
+  /etc/systemd/system/uncloud.service
+```
+
+Stop the daemon before starting the socket unit, then reload systemd, enable the socket, and start the upgraded daemon:
+
+```bash
+sudo systemctl stop uncloud.service && \
+sudo systemctl daemon-reload && \
+sudo systemctl enable uncloud.socket && \
+sudo systemctl start uncloud.socket && \
+sudo systemctl start uncloud.service
+```
+
+### After upgrading
+
+Check your CLI and daemon versions:
+
+```shell
+uc version
+uc machine ls
+```

website/docs/2-getting-started/2-deploy-demo-app.md

@@ -156,7 +156,7 @@ Waiting for the machine to be ready...
 Reserved cluster domain: sh8hsb.uncld.dev
 [+] Deploying service caddy 2/2
  ✔ Container caddy-d7uk on machine-dc3c     Running                                           11.1s
-   ✔ Image caddy:2.11.4 on machine-dc3c     Pulled                                             3.7s
+   ✔ Image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3 on machine-dc3c     Pulled                   3.7s
 
 Updating cluster domain records in Uncloud DNS to point to machines running caddy service...
 [+] Verifying internet access to caddy service 1/1
@@ -216,9 +216,9 @@ uc ls
 ```
 
 ```
-NAME         MODE         REPLICAS   IMAGE                          ENDPOINTS
-caddy        global       1          caddy:2.11.4
-excalidraw   replicated   1          excalidraw/excalidraw:latest   https://excalidraw.sh8hsb.uncld.dev → :80
+NAME         MODE         REPLICAS   IMAGE                                    ENDPOINTS
+caddy        global       1          ghcr.io/unlabs-dev/caddy-uncloud:0.1.3
+excalidraw   replicated   1          excalidraw/excalidraw:latest             https://excalidraw.sh8hsb.uncld.dev → :80
 ```
 
 You can see `caddy` service listed here. That's your reverse proxy, running as a regular Uncloud service.

website/docs/3-concepts/2-ingress/1-overview.md

@@ -25,10 +25,6 @@ When you [publish a service port](2-publishing-services.md), Uncloud automatical
 For advanced use cases, Uncloud allows to customise the Caddy config using the `x-caddy` extension in Compose files.
 See [Custom Caddy configuration](2-publishing-services.md#custom-caddy-configuration) for details.
 
-:::tip Shared TLS certificates
-
-Uncloud also provides native cluster storage for Caddy. It shares TLS certificates and ACME challenge tokens across
+Uncloud configures Caddy to use cluster storage by default. It shares TLS certificates and ACME challenge tokens across
 machines and coordinates certificate issuance. See [Cluster storage for Caddy](4-cluster-storage-for-caddy.md)
-to enable it.
-
-:::
+for details.

website/docs/3-concepts/2-ingress/2-publishing-services.md

@@ -132,8 +132,8 @@ Use it when you need:
 - Advanced TLS settings.
 - Other Caddy features and plugins.
 
-See [Deploying or updating Caddy](3-managing-caddy.md#deploying-or-updating-caddy) for details on deploying Caddy with a
-custom global configuration.
+See [Custom global configuration](3-managing-caddy.md#custom-global-configuration) to learn how to deploy Caddy
+with a custom global config.
 
 ### Templates
 
@@ -216,6 +216,10 @@ Example output:
 # Docs: https://uncloud.run/docs/concepts/ingress/overview
 
 # User-defined global config from service 'caddy'.
+{
+	storage uncloud
+}
+
 *.example.com {
 	tls {
 		dns cloudflare {env.CLOUDFLARE_API_TOKEN}
@@ -274,7 +278,7 @@ example.com {
 The generated config combines:
 
 - Global Caddy configuration (`x-caddy` from the `caddy` service).
-  See [Deploying or updating Caddy](3-managing-caddy.md#deploying-or-updating-caddy) for details.
+  See [Custom global configuration](3-managing-caddy.md#custom-global-configuration) for details.
 - Auto-generated configs from published service ports (`x-ports`).
 - Custom Caddy configs from services (`x-caddy`).
 - Skipped invalid configs with error messages as comments.

website/docs/3-concepts/2-ingress/3-managing-caddy.md

@@ -1,11 +1,22 @@
 # Managing Caddy
 
-Caddy is automatically deployed as a global service `caddy` when you initialise a cluster with `uc machine init`. By
-default, it runs on every machine to handle incoming HTTP/HTTPS traffic and route it to your services.
+Uncloud automatically deploys Caddy as a global service named `caddy` when you initialise a cluster with
+`uc machine init`. By default, it runs on every machine and routes incoming HTTP/HTTPS traffic to your services.
 
-Since [v0.21.0](https://github.com/psviderski/uncloud/releases/tag/v0.21.0), Uncloud provides native
-[cluster storage for Caddy](4-cluster-storage-for-caddy.md) (opt-in) to share TLS certificates and coordinate
-certificate issuance across machines.
+Since [v0.22.0](https://github.com/psviderski/uncloud/releases/tag/v0.22.0), the default Caddy deployment uses the
+Uncloud Caddy image
+[`ghcr.io/unlabs-dev/caddy-uncloud`](https://github.com/unlabs-dev/caddy-uncloud/pkgs/container/caddy-uncloud) with
+[cluster storage](4-cluster-storage-for-caddy.md) to share TLS certificates across machines. This image is a Caddy build
+that includes the [Uncloud storage module](https://github.com/unlabs-dev/caddy-uncloud) and a few other popular modules
+such as [caddy-l4](https://github.com/mholt/caddy-l4) and [cloudflare](https://github.com/caddy-dns/cloudflare).
+
+:::warning
+
+If you want to upgrade or redeploy Caddy on an existing cluster created with Uncloud v0.21.0 or earlier, follow
+[Upgrading from earlier versions](4-cluster-storage-for-caddy.md#upgrading-from-earlier-versions) before redeploying
+Caddy.
+
+:::
 
 ## Checking status
 
@@ -16,55 +27,92 @@ uc inspect caddy
 ```
 
 ```
-ID:    b5b269d5dc5ed4fdae6542894f94de82
-Name:  caddy
-Mode:  global
+Service ID: b5b269d5dc5ed4fdae6542894f94de82
+Name:       caddy
+Mode:       global
+Labels:
+  uncloud.caddy.managed=
 
-CONTAINER ID   IMAGE          CREATED        STATUS       MACHINE
-fb8f390e634d   caddy:2.10.0   3 weeks ago    Up 3 weeks   prod-ap1
-0182f5d7bd9f   caddy:2.10.0   3 months ago   Up 3 weeks   prod-us1
+CONTAINER ID   IMAGE                                    CREATED        STATUS       IP ADDRESS   MACHINE
+fb8f390e634d   ghcr.io/unlabs-dev/caddy-uncloud:0.1.4   3 weeks ago    Up 3 weeks   10.210.1.2   prod-ap1
+0182f5d7bd9f   ghcr.io/unlabs-dev/caddy-uncloud:0.1.4   3 weeks ago    Up 3 weeks   10.210.0.2   prod-us1
+```
+
+## Verifying config
+
+View the complete generated Caddyfile that the `caddy` service uses. It includes global configuration, sites for
+published service ports, custom service configs, and comments explaining any skipped invalid configs. It also warns if
+the last config load failed. Use `--machine NAME` to check a specific machine.
+
+```shell
+uc caddy config
+```
+
+See [Verifying Caddy config](2-publishing-services.md#verifying-caddy-config) for an example output and details on how
+it's generated.
+
+## Listing certificates
+
+List the issued certificates in cluster storage:
+
+```shell
+uc caddy cert ls
+```
+
+```
+ID             NAME                            ISSUER                    EXPIRES
+c111e23615f7   dns.uncloud.run                 Let's Encrypt             2026-12-31 (2 months)
+b5dcd2a03e1b   nginx.2t5ex2.uncld.dev          Let's Encrypt             2026-12-31 (2 months)
+70707fd2fea2   test-staging.2t5ex2.uncld.dev   Let's Encrypt (staging)   2026-12-31 (2 months)
+c5f9301e1c06   uncloud.run                     Let's Encrypt             2026-12-31 (2 months)
 ```
 
 ## Deploying or updating Caddy
 
-### Using CLI
-
-Update to the latest stable version using the [caddy](https://hub.docker.com/_/caddy) image from Docker Hub:
+Deploy Caddy or upgrade it to the latest version of the Uncloud Caddy image:
 
 ```shell
 uc caddy deploy
 ```
 
-Deploy a specific version or custom image:
+The command shows a deployment plan and asks for confirmation, then performs a rolling update.
+
+You can set a specific image, global Caddyfile, or limit deployment to specific machines:
 
 ```shell
-uc caddy deploy --image caddybuilds/caddy-cloudflare:2.11.4
+# Deploy a specific version or custom image and keep it for later deployments.
+uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.4
+
+# Deploy with a custom global Caddy config and only on selected machines.
+uc caddy deploy --caddyfile global.Caddyfile --machine machine1,machine2
 ```
 
-Deploy only to a specific machine or a subset of machines (comma-separated list):
+The command remembers these values for later deployments. Pass an empty value to reset a flag to its default:
 
 ```shell
-uc caddy deploy --machine machine1
-uc caddy deploy --machine machine2,machine3,machine4
+uc caddy deploy --image "" --caddyfile "" --machine ""
 ```
 
-Deploy with custom global configuration:
+See [`uc caddy deploy`](../../9-cli-reference/uc_caddy_deploy.md) for all flags and examples.
 
-```shell
-uc caddy deploy --caddyfile global.Caddyfile
-```
+## Custom global configuration
 
-Example global configuration:
+A custom Caddyfile replaces the default global config. Uncloud combines this config with the sites generated from
+[published ports](2-publishing-services.md) and
+[custom service configs](2-publishing-services.md#custom-caddy-configuration). Include `storage uncloud` to keep using
+[cluster storage](4-cluster-storage-for-caddy.md):
 
-```caddyfile title=global.Caddyfile
+```caddyfile title="global.Caddyfile"
 # Global options.
 {
-    debug
+    storage uncloud
+    # Uncomment for troubleshooting:
+    # debug
 }
 
 # A snippet that can be reused in custom Caddy configs for services (x-caddy).
 (my_snippet) {
-    ...
+    # ...
 }
 
 # Expose an internal service that is not managed by Uncloud.
@@ -73,10 +121,29 @@ internal.example.com {
 }
 ```
 
-### Using Compose
+Deploy your custom global Caddy config:
 
-You can manage the Caddy deployment with a Compose file for more control. For example, to deploy a custom global Caddy
-config that uses the DNS challenge with Cloudflare to obtain a wildcard TLS certificate for `*.example.com`:
+```shell
+uc caddy deploy --caddyfile global.Caddyfile
+```
+
+## Custom image
+
+A custom image must include the [Uncloud storage module](https://github.com/unlabs-dev/caddy-uncloud) to use cluster
+storage. It must also include `curl` for the health check and start Caddy with `/etc/caddy/Caddyfile`.
+
+The default Uncloud Caddy image
+[`ghcr.io/unlabs-dev/caddy-uncloud`](https://github.com/unlabs-dev/caddy-uncloud/pkgs/container/caddy-uncloud) satisfies
+these requirements. See the [release notes](https://github.com/unlabs-dev/caddy-uncloud/releases/latest) for the Caddy
+version and the list of included modules.
+
+If you need other Caddy modules or want to choose a compatible Caddy version, you can
+[build and deploy your own image](https://github.com/unlabs-dev/caddy-uncloud#building-and-deploying-a-custom-image).
+
+## Managing Caddy with Compose
+
+Use Compose when you need more control, such as environment variables for a DNS provider. This example uses the
+Cloudflare module included in the default image to obtain a wildcard certificate for `*.example.com`:
 
 <Tabs>
 <TabItem value="compose.yaml">
@@ -84,13 +151,11 @@ config that uses the DNS challenge with Cloudflare to obtain a wildcard TLS cert
 ```yaml
 services:
   caddy:
-    image: caddybuilds/caddy-cloudflare:2.11.4
+    image: ghcr.io/unlabs-dev/caddy-uncloud:0.1.4
     environment:
       # unix// is not a typo. Caddy uses network/address format, not a unix:// URL.
       CADDY_ADMIN: unix//run/caddy/admin.sock
-    env_file:
-      # Contains CLOUDFLARE_API_TOKEN=xxxxx
-      - .env.secrets
+      CLOUDFLARE_API_TOKEN: secret://cloudflare_api_token
     healthcheck:
       test: curl -fsS -o /dev/null --unix-socket /run/caddy/admin.sock http://localhost/config/
       interval: 30s
@@ -102,22 +167,26 @@ services:
       - /var/lib/uncloud/caddy:/etc/caddy:ro
       # Persists TLS certificates and other assets when using Caddy's default local storage.
       - /var/lib/uncloud/caddy:/data
-      - /run/uncloud/caddy:/run/caddy
-      # Required by the Uncloud cluster storage module (https://github.com/unlabs-dev/caddy-uncloud).
-      # Remove this mount if cluster storage is not used for Caddy.
+      # Uncloud API socket is required by the cluster storage module (https://github.com/unlabs-dev/caddy-uncloud).
       # Mount the directory, not the socket file, so Caddy sees a replacement socket after the daemon restarts.
       - /run/uncloud/api:/run/uncloud/api:ro
+      - /run/uncloud/caddy:/run/caddy
     x-ports:
       - 80:80@host
       - 443:443@host
       - 443:443/udp@host
-    x-caddy: Caddyfile
+    x-caddy: ./Caddyfile
     deploy:
       mode: global
     # Optional: deploy only to specific machines.
     # x-machines:
     #   - machine1
     #   - machine2
+
+secrets:
+  cloudflare_api_token:
+    # Read the secret value from 1Password by running this command.
+    x-command: op read "op://prod/caddy/cloudflare_api_token"
 ```
 
 </TabItem>
@@ -126,144 +195,34 @@ services:
 ```caddyfile
 # Global options.
 {
-	debug
+    storage uncloud
 }
 
-# A snippet that can be reused in custom Caddy configs for services (x-caddy).
-(my_snippet) {
-	...
-}
-
-# Obtain a wildcard TLS certificate for all subdomains of example.name using DNS challenge with Cloudflare.
-# It will be used for services that publish ports with hostnames under example.name.
+# Obtain a wildcard TLS certificate for all subdomains of example.com using DNS challenge with Cloudflare.
+# It will be used for services that publish ports with hostnames under example.com.
 *.example.com {
-	tls {
-		dns cloudflare {env.CLOUDFLARE_API_TOKEN}
-	}
-	respond "No host matched" 404
-}
-
-# Expose an internal service that is not managed by Uncloud.
-internal.example.com {
-	reverse_proxy 192.168.1.100
+    tls {
+        dns cloudflare {env.CLOUDFLARE_API_TOKEN}
+    }
+    respond "No host matched" 404
 }
 ```
 
 </TabItem>
 </Tabs>
 
-:::info note
-
-The specified `environment`, `volumes`, and `x-ports` properties are essential for Caddy to function correctly in the
-Uncloud cluster. Do not change the source paths of the volume mounts as the Uncloud daemon relies on them to communicate
-with Caddy and update its configuration.
-
-The image must include `curl` for the healthcheck and start Caddy with `/etc/caddy/Caddyfile`. Images based on the
-official [Caddy image](https://hub.docker.com/_/caddy) do both by default.
-
-Keep the `/data` mount while using Caddy's default local storage so TLS certificates survive container updates. You can
-remove this mount when using [cluster storage for Caddy](4-cluster-storage-for-caddy.md).
-
-:::
-
 Deploy or update the `caddy` service from the Compose file:
 
 ```shell
 uc deploy
 ```
 
-## Verifying config
+:::info note
 
-View the complete generated Caddyfile served by the `caddy` service. This is useful for debugging and verifying custom
-global and service-specific Caddy configs.
+Keep the `environment`, `volumes`, and `x-ports` properties as shown, including the source paths of the volume mounts.
+The Uncloud daemon and the storage module rely on them.
 
-```shell
-uc caddy config
-```
+:::
 
-Example output:
-
-```caddyfile
-# Caddyfile autogenerated by Uncloud (DO NOT EDIT): 2025-12-22T10:30:12Z
-# Automatically updated on service or health status changes.
-# Docs: https://uncloud.run/docs/concepts/ingress/overview
-
-# User-defined global config from service 'caddy'.
-# Global options.
-{
-	debug
-}
-
-# A snippet that can be reused in custom Caddy configs for services (x-caddy).
-(my_snippet) {
-	...
-}
-
-# Obtain a wildcard TLS certificate for all subdomains of example.name using DNS challenge with Cloudflare.
-# It will be used for services that publish ports with hostnames under example.name.
-*.example.com {
-	tls {
-		dns cloudflare {env.CLOUDFLARE_API_TOKEN}
-	}
-	respond "No host matched" 404
-}
-
-# Expose an internal service that is not managed by Uncloud.
-internal.example.com {
-	reverse_proxy 192.168.1.100
-}
-
-# Health check endpoint to verify Caddy reachability on this machine.
-http:// {
-	handle /.uncloud-verify {
-		respond "a369b9388812f9557feef6a0f5b46f2e" 200
-	}
-	log
-}
-
-(common_proxy) {
-	# Retry failed requests up to lb_retries times against other available upstreams.
-	lb_retries 3
-	# Upstreams are marked unhealthy for fail_duration after a failed request (passive health checking).
-	fail_duration 30s
-}
-
-# Sites generated from service ports.
-
-https://app.example.com {
-	reverse_proxy 10.210.1.3:8000 10.210.2.5:8000 {
-		import common_proxy
-	}
-	log
-}
-
-https://api.example.com {
-	reverse_proxy 10.210.2.2:9000 10.210.1.7:9000 10.210.2.3:9000 {
-		import common_proxy
-	}
-	log
-}
-
-# User-defined config for service 'web'.
-www.example.com {
-	redir https://example.com{uri} permanent
-}
-
-example.com {
-	reverse_proxy 10.210.0.3:8000 {
-		import common_proxy
-	}
-	log
-}
-
-# Skipped invalid user-defined configs:
-# - service 'duplicate-hostname': validation failed: adapting config using caddyfile adapter: ambiguous site definition: example.com
-# - service 'invalid': validation failed: adapting config using caddyfile adapter: Caddyfile:61: unrecognized directive: invalid_directive
-```
-
-The generated config combines:
-
-- Global Caddy configuration (`x-caddy` from the `caddy` service).
-- Auto-generated configs from published service ports (`x-ports`).
-- Custom Caddy configs from services (`x-caddy`).
-- Skipped invalid configs with error messages as comments.
+Continue using `uc deploy` for a Compose-managed Caddy service. Switching to `uc caddy deploy` replaces its
+configuration, including environment variables and custom mounts.

website/docs/3-concepts/2-ingress/4-cluster-storage-for-caddy.md

@@ -1,73 +1,120 @@
 # Cluster storage for Caddy
 
+Starting with Uncloud [v0.22.0](https://github.com/psviderski/uncloud/releases/tag/v0.22.0), Caddy uses cluster storage
+by default.
+
 The [Uncloud storage module](https://github.com/unlabs-dev/caddy-uncloud) lets Caddy instances share TLS certificates,
-private keys, and ACME challenge tokens through Uncloud's cluster store. It uses distributed locks to coordinate
-certificate issuance. Once one instance obtains a certificate, the others can use it too.
+private keys, and ACME challenge tokens through Uncloud's cluster store.
 
-## Why use cluster storage?
+This matters when you run Caddy on multiple machines and distribute traffic between them using DNS or a load balancer.
+In this case, an ACME challenge can reach a different Caddy instance from the one requesting the certificate. Cluster
+storage lets any Caddy instance read the challenge token from the shared cluster store and complete the ACME challenge.
+It uses distributed locks to coordinate certificate issuance. Once one instance obtains a certificate, the others can
+use it too.
 
-By default, each Caddy instance stores its certificates locally. When DNS points to multiple machines or a load balancer
-distributes traffic between them, an ACME challenge can reach a different instance from the one requesting the
-certificate. That instance may not have the challenge token, which can delay or prevent certificate issuance.
+## Upgrading from earlier versions
 
-Cluster storage is optional. The default Caddy image does not include the module, so you need to deploy an image that
-includes it and configure Caddy to use it.
+Starting with Uncloud v0.22.0, `uc caddy deploy` deploys the Uncloud Caddy image ([
+`ghcr.io/unlabs-dev/caddy-uncloud`](https://github.com/unlabs-dev/caddy-uncloud/pkgs/container/caddy-uncloud)) with
+cluster storage enabled by default.
 
-## Enabling cluster storage
+Earlier versions deployed the official [`caddy`](https://hub.docker.com/_/caddy) image with local certificate storage.
+Upgrading Uncloud does not change the running Caddy service. Redeploy it to switch to the new default image and cluster
+storage.
 
-:::warning Certificate migration
+Before you deploy, upgrade the [`uc` CLI](../../2-getting-started/1-install-cli.md) and the daemon on all machines to
+v0.22.0 or newer. Follow the [upgrade instructions](https://github.com/psviderski/uncloud/releases/latest) for daemon
+upgrades, including any required migrations from older releases. Check current versions with `uc version` and
+`uc machine ls`.
 
-Caddy doesn't migrate existing certificates automatically when you change its storage backend. **All certificates will
-need to be reissued unless you migrate your existing storage first.** Use the experimental
-[`caddy storage export` and `caddy storage import`](https://caddyserver.com/docs/command-line#caddy-storage)
-commands with the old and new configs to transfer the storage contents.
+The first `uc caddy deploy` after the upgrade doesn't know the values you set with earlier versions of `uc`. It replaces
+your image, global config, and machine list with the defaults unless you pass them as flags. If you deployed Caddy only
+to some machines, add `--machine` with the same machines to every `uc caddy deploy` command below.
 
-:::
-
-:::info Requirements
-
-Cluster storage requires Uncloud **v0.21.0 or newer** for both the `uc` CLI and the daemon on every cluster machine.
-Check your CLI version with `uc version` and daemon versions with `uc machine ls`. Upgrade older versions before
-enabling cluster storage.
-
-:::
-
-Create a global Caddyfile, or add `storage uncloud` to the global options in your existing one:
-
-```caddyfile title="global.Caddyfile"
-{
-    storage uncloud
-
-    # Uncomment to enable debug logs useful for troubleshooting storage operations:
-    # debug
-}
-```
-
-Deploy Caddy with the pre-built module image and your global config:
+If you already use `storage uncloud`, your certificates stay where they are. Run `uc caddy deploy` and pass the global
+config you deployed it with, unless it only contains `storage uncloud`:
 
 ```shell
-uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3 --caddyfile global.Caddyfile
+uc caddy deploy --caddyfile global.Caddyfile
 ```
 
-See the [module README](https://github.com/unlabs-dev/caddy-uncloud#usage) for custom image builds, Compose deployment,
-and additional storage options.
+### Quick upgrade
 
-## Verifying storage
+Caddy does not copy certificates when you change storage backends. Unless you migrate them first, Caddy will reissue
+them. HTTPS may fail for those domains until that finishes, and a large number of certificates can hit
+[Let's Encrypt rate limits](https://letsencrypt.org/docs/rate-limits/).
 
-Check that `storage uncloud` appears in the Caddy config:
+Run the command below if you don't mind reissuing all certificates:
 
 ```shell
-uc caddy config
+uc caddy deploy
 ```
 
-List issued certificates in cluster storage with [`uc caddy cert ls`](../../9-cli-reference/uc_caddy_cert_ls.md):
+### Migrating certificates
 
-```shell
-uc caddy cert ls
+Follow these steps to keep your existing certificates and avoid HTTPS failures:
 
-ID             NAME                            ISSUER                    EXPIRES
-c111e23615f7   dns.uncloud.run                 Let's Encrypt             2026-12-31 (2 months)
-b5dcd2a03e1b   nginx.2t5ex2.uncld.dev          Let's Encrypt             2026-12-31 (2 months)
-70707fd2fea2   test-staging.2t5ex2.uncld.dev   Let's Encrypt (staging)   2026-12-31 (2 months)
-c5f9301e1c06   uncloud.run                     Let's Encrypt             2026-12-31 (2 months)
-```
+1. Deploy the new image but stay on local storage. Use your existing global config if you have one, or an empty file:
+
+   ```shell
+   touch local.Caddyfile
+   uc caddy deploy --caddyfile local.Caddyfile
+   ```
+
+2. Find the ID of the Caddy container on each machine:
+
+   ```shell
+   uc inspect caddy
+   ```
+
+3. Copy the certificates to cluster storage by running this command for each container ID. Each machine stores its own
+   certificates, so don't skip any of them. These
+   [`caddy storage`](https://caddyserver.com/docs/command-line#caddy-storage) commands are experimental.
+
+   ```shell
+   uc exec -T --container CONTAINER_ID caddy sh -c '
+     printf "{\n\tstorage uncloud\n}\n" > /tmp/Caddyfile.cluster
+     caddy storage export --config /etc/caddy/Caddyfile --output - |
+       caddy storage import --config /tmp/Caddyfile.cluster --input -
+     rm /tmp/Caddyfile.cluster
+   '
+   ```
+
+4. Check that all your certificates are in cluster storage:
+
+   ```shell
+   uc caddy cert ls
+   ```
+
+5. Switch Caddy to cluster storage. If you have a custom global config, add `storage uncloud` to it and pass it with
+   `--caddyfile global.Caddyfile` instead. The command warns that all certificates will be reissued. You can ignore this
+   because you've already copied them.
+
+   ```shell
+   uc caddy deploy --caddyfile ""
+   ```
+
+6. Check that the Caddy config includes `storage uncloud` and doesn't show any config load warnings:
+
+   ```shell
+   uc caddy config
+   ```
+
+### Upgrading a Compose deployment
+
+If you manage Caddy with Compose, keep using `uc deploy` and update your Compose file to match the
+[Compose example](3-managing-caddy.md#managing-caddy-with-compose). To keep your certificates, follow the
+[migration steps](#migrating-certificates) with these changes:
+
+- In step 1, update the image and add the `/run/uncloud/api:/run/uncloud/api:ro` volume mount but leave out
+  `storage uncloud` in your global Caddy config (`x-caddy`). Then run `uc deploy`.
+- In step 5, add `storage uncloud` to your global Caddy config, then run `uc deploy`.
+
+## Keeping local storage
+
+To keep local storage, pass a custom Caddyfile without `storage uncloud` using `--caddyfile`. The default Uncloud Caddy
+image supports local storage too. If you use the official `caddy` image, you must also provide a Caddyfile without
+`storage uncloud`.
+
+Deployments using `uc caddy deploy` keep the `/data` mount so local certificates survive container updates. For Compose
+deployments using local storage, keep or add `/var/lib/uncloud/caddy:/data`.

website/docs/9-cli-reference/uc_caddy_cert_ls.md

@@ -6,11 +6,14 @@ List certificates in cluster storage for Caddy.
 
 List certificates stored in the Uncloud cluster storage for Caddy.
 
-Caddy must use the Uncloud storage module (https://github.com/unlabs-dev/caddy-uncloud)
-configured with 'storage uncloud' in the global options for its certificates to appear here.
+Caddy uses cluster storage by default. Certificates don't appear here if Caddy uses local storage,
+for example, if it runs with a custom global config without 'storage uncloud' or was deployed with
+uc v0.21.0 or earlier and not upgraded.
 
 This inventory does not check whether Caddy currently serves or trusts a certificate.
 
+See https://uncloud.run/docs/concepts/ingress/cluster-storage-for-caddy for details.
+
 ```
 uc caddy cert ls [flags]
 ```

website/docs/9-cli-reference/uc_caddy_deploy.md

@@ -7,17 +7,59 @@ Deploy or upgrade Caddy reverse proxy across all machines in the cluster.
 Deploy or upgrade Caddy reverse proxy across all machines in the cluster.
 A rolling update is performed when updating existing containers to minimise disruption.
 
+By default, this deploys the latest version of the Uncloud Caddy image with a global config that enables cluster
+storage for TLS certificates. See https://github.com/unlabs-dev/caddy-uncloud for more details.
+
+Values set with --image, --caddyfile, and --machine are kept for later deployments. Pass an empty value to reset
+a flag to its default.
+
+If this command didn't deploy the current caddy service, for example, it was deployed with 'uc deploy'
+from a Compose file or an older version of uc, the deployment replaces its configuration.
+
+Requires Uncloud daemon 0.22.0 or newer on all machines.
+
 ```
 uc caddy deploy [flags]
 ```
 
+## Examples
+
+```
+  # Deploy Caddy or upgrade it to the latest version of the Uncloud Caddy image.
+  uc caddy deploy
+
+  # Deploy a custom image and keep it for later deployments.
+  uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3
+
+  # Go back to the latest version of the Uncloud Caddy image.
+  uc caddy deploy --image ""
+
+  # Deploy a custom image build and a custom global Caddy config and keep both for later deployments.
+  uc caddy deploy --image myorg/caddy-uncloud:latest --caddyfile global.Caddyfile
+
+  # Go back to the default global Caddy config but keep the custom image.
+  uc caddy deploy --caddyfile ""
+
+  # Deploy only to specific machines and keep them for later deployments.
+  uc caddy deploy --machine machine1,machine2
+
+  # Go back to deploying to all machines.
+  uc caddy deploy --machine ""
+```
+
 ## Options
 
 ```
       --caddyfile string   Path to a custom global Caddy config (Caddyfile) that will be prepended to the auto-generated Caddy config.
+                           An empty value resets it to the default config that enables cluster storage.
+                           (default is the current custom config or the default config)
   -h, --help               help for deploy
-      --image string       Caddy Docker image to deploy. (default caddy:LATEST_VERSION)
-  -m, --machine strings    Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated list. (default is all machines)
+      --image string       Caddy Docker image to deploy. It must include the Uncloud storage module if the Caddy config uses it.
+                           An empty value resets it to the latest version of the Uncloud Caddy image.
+                           (default is the current custom image or ghcr.io/unlabs-dev/caddy-uncloud:LATEST_VERSION)
+  -m, --machine strings    Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated list.
+                           An empty value resets it to all machines.
+                           (default is the current machines or all machines)
 ```
 
 ## Options inherited from parent commands

website/docs/9-cli-reference/uc_run.md

@@ -30,11 +30,12 @@ uc run IMAGE [COMMAND...] [flags]
       --privileged                    Give extended privileges to service containers. This is a security risk and should be used with caution.
   -p, --publish strings               Publish a service port to make it accessible outside the cluster. Can be specified multiple times.
                                       Format: [hostname:]container_port[/protocol] or [host_ip|host_prefix:]host_port:container_port[/protocol]@host
-                                      Supported protocols: tcp, udp, http, https (default is tcp). If a hostname for http(s) port is not specified
-                                      and a cluster domain is reserved, service-name.cluster-domain will be used as the hostname.
+                                      Supported protocols: tcp, udp, http, https. The protocol defaults to https if a hostname is specified,
+                                      otherwise tcp. http(s) ports without a hostname use 'service-name.cluster-domain' as the default hostname
+                                      if a cluster domain is reserved.
                                       Examples:
                                         -p 8080/https                       Publish port 8080 as HTTPS via reverse proxy with default service-name.cluster-domain hostname
-                                        -p app.example.com:8080/https       Publish port 8080 as HTTPS via reverse proxy with custom hostname
+                                        -p app.example.com:8080             Publish port 8080 as HTTPS via reverse proxy with custom hostname
                                         -p 53:5353/udp@host                 Bind UDP port 5353 to host port 53
                                         -p 192.168.76.0/24:53:5353/udp@host Bind UDP port 5353 to host port 53 on every host IP address
                                                                             contained in the prefix 192.168.76.0/24

website/docs/9-cli-reference/uc_service_run.md

@@ -30,11 +30,12 @@ uc service run IMAGE [COMMAND...] [flags]
       --privileged                    Give extended privileges to service containers. This is a security risk and should be used with caution.
   -p, --publish strings               Publish a service port to make it accessible outside the cluster. Can be specified multiple times.
                                       Format: [hostname:]container_port[/protocol] or [host_ip|host_prefix:]host_port:container_port[/protocol]@host
-                                      Supported protocols: tcp, udp, http, https (default is tcp). If a hostname for http(s) port is not specified
-                                      and a cluster domain is reserved, service-name.cluster-domain will be used as the hostname.
+                                      Supported protocols: tcp, udp, http, https. The protocol defaults to https if a hostname is specified,
+                                      otherwise tcp. http(s) ports without a hostname use 'service-name.cluster-domain' as the default hostname
+                                      if a cluster domain is reserved.
                                       Examples:
                                         -p 8080/https                       Publish port 8080 as HTTPS via reverse proxy with default service-name.cluster-domain hostname
-                                        -p app.example.com:8080/https       Publish port 8080 as HTTPS via reverse proxy with custom hostname
+                                        -p app.example.com:8080             Publish port 8080 as HTTPS via reverse proxy with custom hostname
                                         -p 53:5353/udp@host                 Bind UDP port 5353 to host port 53
                                         -p 192.168.76.0/24:53:5353/udp@host Bind UDP port 5353 to host port 53 on every host IP address
                                                                             contained in the prefix 192.168.76.0/24