feat(caddy): make cluster storage for Caddy default, remember 'uc caddy deploy' options

2026-11-10 12:44:16 +00:00 · 2026-11-05 16:03:34 +10:00
parent 76b04dd56c
commit 22b1c68798
13 changed files with 1173 additions and 98 deletions
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"regexp"
+	"strings"
 	"time"

 	"github.com/Masterminds/semver"
@@ -11,6 +12,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/psviderski/uncloud/api/pb"
+	"github.com/psviderski/uncloud/internal/version"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client/deploy"
 	"google.golang.org/protobuf/types/known/emptypb"
@@ -18,11 +20,33 @@ import (

 const (
 	CaddyServiceName = "caddy"
-	// CaddyImage is the official Caddy Docker image on Docker Hub: https://hub.docker.com/_/caddy
-	CaddyImage = "caddy"
+	// CaddyImage is the Uncloud Caddy image that is deployed by default. It includes the Uncloud storage module for
+	// sharing TLS certificates across machines: https://github.com/unlabs-dev/caddy-uncloud
+	CaddyImage = "ghcr.io/unlabs-dev/caddy-uncloud"
+	// DefaultCaddyConfig is the default global Caddyfile. It must be compatible with all versions of CaddyImage allowed
+	// by caddyImageConstraint.
+	DefaultCaddyConfig = "{\n\tstorage uncloud\n}"
+	// CaddyStorageFileSystem is the name of the default Caddy storage module that stores TLS assets locally.
+	CaddyStorageFileSystem = "file_system"
+	// CaddyStorageUncloud is the name of the Caddy storage module that stores TLS assets in the cluster.
+	CaddyStorageUncloud = "uncloud"
+	// MinCaddyDaemonVersion is the minimum daemon version required for 'uc caddy deploy'. Older daemons don't store
+	// service labels that mark the service as managed by this command. Distributed locks were introduced in 0.21.0,
+	// which is the minimum required by the cluster storage even on machines that don't run the Caddy service.
+	MinCaddyDaemonVersion = "0.22.0"
+
+	// officialCaddyImage is the official Caddy Docker image on Docker Hub: https://hub.docker.com/_/caddy
+	officialCaddyImage = "docker.io/library/caddy"
 )

-var caddyImageTagRegex = regexp.MustCompile(`^2\.\d+\.\d+$`)
+var (
+	// caddyImageConstraint limits CaddyImage to the major version that is known to be compatible with
+	// DefaultCaddyConfig.
+	caddyImageConstraint = mustNewConstraint("<1.0.0")
+	// caddyImageTagRegex matches the exact version tags of CaddyImage, skipping floating tags and image variants.
+	caddyImageTagRegex = regexp.MustCompile(`^\d+\.\d+\.\d+$`)
+	caddyStorageRegex  = regexp.MustCompile(`(?m)^[ \t]*storage[ \t]+(\S+)`)
+)

 // CaddyClient provides Caddy operations over its parent Client's connection.
 // The parent client owns the connection and must be used to close it.
@@ -64,19 +88,28 @@ func (c *CaddyClient) Config(ctx context.Context, opts CaddyConfigOptions) (api.
 	return config, nil
 }

-// CaddyDeploymentOptions configures a Caddy reverse proxy deployment.
+// CaddyDeploymentOptions configures a Caddy reverse proxy deployment. The zero value deploys the latest version of
+// CaddyImage with DefaultCaddyConfig to all machines.
 type CaddyDeploymentOptions struct {
-	// Image defaults to the latest stable 2.x.x official Caddy image.
-	// Custom images must include curl and start Caddy with /etc/caddy/Caddyfile.
+	// Image defaults to the latest version of CaddyImage. A non-empty value is marked as custom.
+	// Custom images must include curl and start Caddy with /etc/caddy/Caddyfile. They must also include the Uncloud
+	// storage module if the config uses 'storage uncloud'.
 	Image string
-	// Config contains an optional global Caddyfile.
-	Config    string
+	// Config is the global Caddyfile. Nil means DefaultCaddyConfig. Any other value, even empty, is marked as custom.
+	Config    *string
 	Placement api.Placement
 }

-// NewDeployment creates a new deployment for a Caddy reverse proxy service.
-// The service is deployed in global mode to all machines in the cluster. If the image is not provided, the latest
-// version of the official Caddy Docker image is used.
+// ConfigOrDefault returns the global Caddyfile of the deployment.
+func (o CaddyDeploymentOptions) ConfigOrDefault() string {
+	if o.Config != nil {
+		return *o.Config
+	}
+	return DefaultCaddyConfig
+}
+
+// NewDeployment creates a new managed deployment for a Caddy reverse proxy service.
+// The service is deployed in global mode to all machines in the cluster or the machines set in the placement.
 func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOptions) (*deploy.Deployment, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
@@ -90,6 +123,10 @@ func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOpt

 		image = reference.FamiliarString(latest)
 	}
+	config := opts.ConfigOrDefault()
+	if err := validateCaddyImageStorage(image, config); err != nil {
+		return nil, err
+	}

 	spec := api.ServiceSpec{
 		Container: api.ContainerSpec{
@@ -186,17 +223,119 @@ func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOpt
 		},
 	}

-	if opts.Config != "" {
+	if config != "" {
 		spec.Caddy = &api.CaddySpec{
-			Config: opts.Config,
+			Config: config,
 		}
 	}

+	spec.Labels = map[string]string{api.LabelCaddyManaged: ""}
+	if opts.Image != "" {
+		spec.Labels[api.LabelCaddyImageCustom] = ""
+	}
+	if opts.Config != nil {
+		spec.Labels[api.LabelCaddyConfigCustom] = ""
+	}
+
 	return c.client.NewDeployment(spec, nil), nil
 }

-// latestCaddyImage returns the latest image of the official Caddy Docker image on Docker Hub.
-// The latest image is determined by the latest version tag 2.x.x.
+// CheckDaemonVersions returns an error naming the available machines whose daemons are older than
+// MinCaddyDaemonVersion. Machines that are down are skipped because a deployment doesn't touch them.
+func (c *CaddyClient) CheckDaemonVersions(ctx context.Context) error {
+	machines, err := c.client.ListMachines(ctx, &api.MachineFilter{Available: true})
+	if err != nil {
+		return fmt.Errorf("list machines: %w", err)
+	}
+
+	if outdated := outdatedCaddyDaemons(machines); len(outdated) > 0 {
+		return fmt.Errorf("deploying Caddy requires Uncloud daemon %s or newer on all machines, upgrade these "+
+			"machines and try again: %s. See https://github.com/psviderski/uncloud/releases/latest",
+			MinCaddyDaemonVersion, strings.Join(outdated, ", "))
+	}
+	return nil
+}
+
+// outdatedCaddyDaemons returns descriptions of the machines with outdated daemons. Prereleases and development builds
+// aren't considered outdated because their versions don't show whether they're older than a stable release.
+func outdatedCaddyDaemons(machines api.MachineMembersList) []string {
+	minVersion := semver.MustParse(MinCaddyDaemonVersion)
+
+	var outdated []string
+	for _, m := range machines {
+		v, err := semver.NewVersion(m.Machine.DaemonVersion)
+		if err != nil || v.Prerelease() != "" {
+			continue
+		}
+		if v.LessThan(minVersion) {
+			outdated = append(outdated, fmt.Sprintf("%s (%s)", m.Machine.Name, m.Machine.DaemonVersion))
+		}
+	}
+
+	return outdated
+}
+
+// CaddyStorageModule returns the name of the storage module set in the global options of a Caddyfile.
+func CaddyStorageModule(config string) string {
+	if match := caddyStorageRegex.FindStringSubmatch(config); match != nil {
+		return match[1]
+	}
+	return CaddyStorageFileSystem
+}
+
+// IsDefaultCaddyImage returns whether the image is any version of CaddyImage.
+func IsDefaultCaddyImage(image string) bool {
+	return imageName(image) == CaddyImage
+}
+
+// IsNewerCaddyImage returns whether the image is a newer version of CaddyImage than the other image.
+// It returns false if any of the images isn't a version of CaddyImage.
+func IsNewerCaddyImage(image, other string) bool {
+	version, otherVersion := caddyImageVersion(image), caddyImageVersion(other)
+	return version != nil && otherVersion != nil && version.GreaterThan(otherVersion)
+}
+
+// caddyImageVersion returns the version in the image tag if the image is a version of CaddyImage, or nil otherwise.
+func caddyImageVersion(image string) *semver.Version {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil || named.Name() != CaddyImage {
+		return nil
+	}
+	tagged, ok := named.(reference.Tagged)
+	if !ok {
+		return nil
+	}
+	version, err := semver.NewVersion(tagged.Tag())
+	if err != nil {
+		return nil
+	}
+
+	return version
+}
+
+// imageName returns the normalised name of the image without its tag or digest.
+// It returns an empty string if the image reference is invalid.
+func imageName(image string) string {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return ""
+	}
+	return named.Name()
+}
+
+// validateCaddyImageStorage rejects the official Caddy image with a config that uses the Uncloud storage module
+// because it doesn't include this module. Otherwise, a new container could start from the previously saved Caddyfile
+// and pass its health check, but its routes would stop updating as Caddy would reject the new config.
+func validateCaddyImageStorage(image, config string) error {
+	if imageName(image) == officialCaddyImage && CaddyStorageModule(config) == CaddyStorageUncloud {
+		return fmt.Errorf("official Caddy image '%s' doesn't support 'storage uncloud' in the global Caddy config: "+
+			"use an image that includes the Uncloud storage module or a Caddyfile without this option. "+
+			"See %s/concepts/ingress/cluster-storage-for-caddy", image, version.DocsURL)
+	}
+	return nil
+}
+
+// latestCaddyImage returns the latest version of CaddyImage allowed by caddyImageConstraint.
 func latestCaddyImage(ctx context.Context) (reference.NamedTagged, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
@@ -222,7 +361,7 @@ func latestCaddyImage(ctx context.Context) (reference.NamedTagged, error) {
 	return imageWithTag, nil
 }

-// latestCaddyTag selects the newest stable 2.x.x tag, falling back to latest.
+// latestCaddyTag selects the newest tag matching caddyImageTagRegex and caddyImageConstraint, falling back to latest.
 func latestCaddyTag(tags []string) string {
 	latestTag := "latest"
 	var latestVersion *semver.Version
@@ -232,7 +371,7 @@ func latestCaddyTag(tags []string) string {
 		}

 		v, err := semver.NewVersion(t)
-		if err != nil {
+		if err != nil || !caddyImageConstraint.Check(v) {
 			continue
 		}
 		if latestVersion == nil || v.GreaterThan(latestVersion) {
@@ -243,3 +382,12 @@ func latestCaddyTag(tags []string) string {

 	return latestTag
 }
+
+// mustNewConstraint parses a version constraint and panics if it's invalid.
+func mustNewConstraint(constraint string) *semver.Constraints {
+	c, err := semver.NewConstraint(constraint)
+	if err != nil {
+		panic(fmt.Sprintf("invalid version constraint '%s': %v", constraint, err))
+	}
+	return c
+}
@@ -5,6 +5,8 @@ import (
 	"testing"

 	"github.com/distribution/reference"
+	"github.com/psviderski/uncloud/api/pb"
+	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -15,7 +17,7 @@ func TestLatestCaddyImage(t *testing.T) {
 	image, err := latestCaddyImage(context.Background())
 	require.NoError(t, err)

-	assert.Regexp(t, `^caddy:2\.\d+\.\d+$`, reference.FamiliarString(image))
+	assert.Regexp(t, `^ghcr\.io/unlabs-dev/caddy-uncloud:0\.\d+\.\d+$`, reference.FamiliarString(image))
 }

 func TestLatestCaddyTag(t *testing.T) {
@@ -27,14 +29,209 @@ func TestLatestCaddyTag(t *testing.T) {
 		want string
 	}{
 		{name: "empty", want: "latest"},
-		{name: "semantic ordering", tags: []string{"2.9.9", "2.11.4", "2.10.0", "2.11.3"}, want: "2.11.4"},
-		{name: "ignore other versions and variants", tags: []string{
-			"latest", "1.0.0", "3.0.0", "2.12.0-rc.1", "2.12.0-alpine", "2.12", "v2.12.0", "2.11.4",
-		}, want: "2.11.4"},
-		{name: "no stable version", tags: []string{"builder", "2.12.0-beta.1"}, want: "latest"},
+		{name: "semantic ordering", tags: []string{"0.1.9", "0.1.11", "0.1.10", "0.1.2"}, want: "0.1.11"},
+		{name: "newest minor version", tags: []string{"0.1.3", "0.2.0", "0.0.9"}, want: "0.2.0"},
+		{name: "ignore newer major versions", tags: []string{"1.0.0", "0.1.3", "2.3.4", "1.0.1"}, want: "0.1.3"},
+		{name: "ignore variants", tags: []string{
+			"latest", "0.1.4-rc.1", "0.1.4-alpine", "0.1", "v0.1.4", "1.0.0-rc.1", "0.1.3",
+		}, want: "0.1.3"},
+		{name: "no stable version", tags: []string{"builder", "0.1.4-beta.1"}, want: "latest"},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			assert.Equal(t, tt.want, latestCaddyTag(tt.tags))
 		})
 	}
 }
+
+func TestDefaultCaddyConfig_UsesUncloudStorage(t *testing.T) {
+	t.Parallel()
+
+	assert.Equal(t, CaddyStorageUncloud, CaddyStorageModule(DefaultCaddyConfig))
+}
+
+func TestCaddyStorageModule(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		name   string
+		config string
+		want   string
+	}{
+		{name: "empty config", want: CaddyStorageFileSystem},
+		{name: "no storage option", config: "{\n\tdebug\n}", want: CaddyStorageFileSystem},
+		{name: "uncloud", config: "{\n\tstorage uncloud\n}", want: CaddyStorageUncloud},
+		{name: "uncloud with options", config: "{\n\tstorage uncloud {\n\t\tfoo bar\n\t}\n}", want: CaddyStorageUncloud},
+		{name: "spaces around", config: "{\n    storage   uncloud  \n}", want: CaddyStorageUncloud},
+		{name: "other module", config: "{\n\tstorage redis\n}", want: "redis"},
+		{name: "explicit file system", config: "{\n\tstorage file_system /data\n}", want: CaddyStorageFileSystem},
+		{name: "commented out", config: "{\n\t# storage uncloud\n}", want: CaddyStorageFileSystem},
+		{name: "similar option", config: "{\n\tstorage_clean_interval 24h\n}", want: CaddyStorageFileSystem},
+		{name: "no module on the same line", config: "{\n\tstorage\n\tdebug\n}", want: CaddyStorageFileSystem},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, CaddyStorageModule(tt.config))
+		})
+	}
+}
+
+func TestIsDefaultCaddyImage(t *testing.T) {
+	t.Parallel()
+
+	assert.True(t, IsDefaultCaddyImage("ghcr.io/unlabs-dev/caddy-uncloud"))
+	assert.True(t, IsDefaultCaddyImage("ghcr.io/unlabs-dev/caddy-uncloud:0.1.3"))
+	assert.False(t, IsDefaultCaddyImage("caddy:2.11.4"))
+	assert.False(t, IsDefaultCaddyImage("example.com/unlabs-dev/caddy-uncloud:0.1.3"))
+	assert.False(t, IsDefaultCaddyImage(""))
+}
+
+func TestIsNewerCaddyImage(t *testing.T) {
+	t.Parallel()
+
+	const repo = "ghcr.io/unlabs-dev/caddy-uncloud"
+	for _, tt := range []struct {
+		name  string
+		image string
+		other string
+		want  bool
+	}{
+		{name: "newer major version", image: repo + ":1.0.0", other: repo + ":0.1.3", want: true},
+		{name: "newer minor version", image: repo + ":0.2.0", other: repo + ":0.1.3", want: true},
+		{name: "same", image: repo + ":0.1.3", other: repo + ":0.1.3"},
+		{name: "older", image: repo + ":0.1.2", other: repo + ":0.1.3"},
+		{name: "different repo", image: "example.com/custom:1.0.0", other: repo + ":0.1.3"},
+		{name: "official image", image: "caddy:2.11.4", other: repo + ":0.1.3"},
+		{name: "tag without version", image: repo + ":latest", other: repo + ":0.1.3"},
+		{name: "no tag", image: repo, other: repo + ":0.1.3"},
+		{name: "fallback to latest tag", image: repo + ":1.0.0", other: repo + ":latest"},
+		{name: "no current image", other: repo + ":0.1.3"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, IsNewerCaddyImage(tt.image, tt.other))
+		})
+	}
+}
+
+func TestCaddyClientNewDeployment(t *testing.T) {
+	t.Parallel()
+
+	const customImage = "example.com/caddy:1.2.3"
+	for _, tt := range []struct {
+		name       string
+		opts       CaddyDeploymentOptions
+		wantConfig string
+		wantLabels map[string]string
+	}{
+		{
+			name:       "defaults",
+			wantConfig: DefaultCaddyConfig,
+			wantLabels: map[string]string{api.LabelCaddyManaged: ""},
+		},
+		{
+			name:       "custom image",
+			opts:       CaddyDeploymentOptions{Image: customImage},
+			wantConfig: DefaultCaddyConfig,
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyImageCustom: ""},
+		},
+		{
+			name:       "custom config",
+			opts:       CaddyDeploymentOptions{Config: new("{\n\tdebug\n}")},
+			wantConfig: "{\n\tdebug\n}",
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
+		},
+		{
+			name:       "custom empty config",
+			opts:       CaddyDeploymentOptions{Config: new("")},
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
+		},
+		{
+			name:       "custom config matching the default is still custom",
+			opts:       CaddyDeploymentOptions{Config: new(DefaultCaddyConfig)},
+			wantConfig: DefaultCaddyConfig,
+			wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
+		},
+		{
+			name: "custom image and config",
+			opts: CaddyDeploymentOptions{
+				Image:  customImage,
+				Config: new("{\n\tdebug\n}"),
+			},
+			wantConfig: "{\n\tdebug\n}",
+			wantLabels: map[string]string{
+				api.LabelCaddyManaged:      "",
+				api.LabelCaddyImageCustom:  "",
+				api.LabelCaddyConfigCustom: "",
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &CaddyClient{client: &Client{}}
+			d, err := c.NewDeployment(context.Background(), tt.opts)
+			require.NoError(t, err)
+
+			assert.Equal(t, tt.wantConfig, d.Spec.CaddyConfig())
+			assert.Equal(t, tt.wantLabels, d.Spec.Labels)
+		})
+	}
+
+	t.Run("official image needs a config without cluster storage", func(t *testing.T) {
+		c := &CaddyClient{client: &Client{}}
+		_, err := c.NewDeployment(context.Background(), CaddyDeploymentOptions{Image: "caddy:2.11.4"})
+		require.ErrorContains(t, err, "doesn't support 'storage uncloud'")
+
+		_, err = c.NewDeployment(context.Background(), CaddyDeploymentOptions{Image: "caddy:2.11.4", Config: new("")})
+		require.NoError(t, err)
+	})
+}
+
+func TestValidateCaddyImageStorage(t *testing.T) {
+	t.Parallel()
+
+	const storageConfig = "{\n\tstorage uncloud\n}"
+	for _, tt := range []struct {
+		name    string
+		image   string
+		config  string
+		wantErr bool
+	}{
+		{name: "official image with uncloud storage", image: "caddy:2.11.4", config: storageConfig, wantErr: true},
+		{name: "official image with registry", image: "docker.io/library/caddy:2", config: storageConfig, wantErr: true},
+		{name: "official image without config", image: "caddy:2.11.4"},
+		{name: "official image with other config", image: "caddy:2.11.4", config: "{\n\tdebug\n}"},
+		{name: "default image with uncloud storage", image: CaddyImage + ":0.1.3", config: storageConfig},
+		{name: "custom image with uncloud storage", image: "example.com/caddy:1", config: storageConfig},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateCaddyImageStorage(tt.image, tt.config)
+			if tt.wantErr {
+				require.ErrorContains(t, err, "doesn't support 'storage uncloud'")
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestOutdatedCaddyDaemons(t *testing.T) {
+	t.Parallel()
+
+	machine := func(name, version string) *pb.MachineMember {
+		return &pb.MachineMember{Machine: &pb.MachineInfo{Name: name, DaemonVersion: version}}
+	}
+
+	assert.Equal(t, []string{"old (0.21.0)", "older (0.20.0)", "unsupported (0.9.3)"},
+		outdatedCaddyDaemons(api.MachineMembersList{
+			machine("old", "0.21.0"),
+			machine("older", "0.20.0"),
+			machine("unsupported", "0.9.3"),
+			machine("no-version", ""),
+			machine("required", "0.22.0"),
+			machine("newer", "0.23.1"),
+			machine("major", "1.0.0"),
+			machine("nightly", "0.22.0-nightly-abc1234"),
+			machine("old-dev-build", "v0.21.0-5-g6c7ddeb-dirty"),
+			machine("local-build", "999.0.0-dev"),
+			machine("not-semver", "6c7ddeb"),
+		}))
+	assert.Empty(t, outdatedCaddyDaemons(api.MachineMembersList{machine("required", "0.22.0")}))
+	assert.Empty(t, outdatedCaddyDaemons(nil))
+}
@@ -134,6 +134,22 @@ func (sp *ServicePlan) Format() string {
 		specTable.Row(mod, "image:", formatImageDiff(oldSpec.Container.Image, sp.Spec.Container.Image))
 	}

+	// An existing service without an old spec only has new containers added to up-to-date ones so its Caddy config and
+	// labels didn't change.
+	specChanged := sp.IsNewService || oldSpec != nil
+	var old api.ServiceSpec
+	if oldSpec != nil {
+		old = *oldSpec
+	}
+
+	// Caddy config row.
+	if specChanged {
+		mod, value := formatCaddyConfigDiff(old.CaddyConfig(), sp.Spec.CaddyConfig(), sp.IsNewService)
+		if value != "" {
+			specTable.Row(mod, "caddy config:", value)
+		}
+	}
+
 	// Replicas row for replicated services.
 	if sp.Spec.Mode == api.ServiceModeReplicated {
 		replicasStr := fmt.Sprintf("%d", sp.Spec.Replicas)
@@ -153,15 +169,11 @@ func (sp *ServicePlan) Format() string {

 	specDiff := specTable.String() + "\n"

-	// Render labels as nested blocks outside the table to not widen its attribute column. An existing service without
-	// an old spec only has new containers added to up-to-date ones so its labels didn't change.
-	if sp.IsNewService || oldSpec != nil {
-		var oldLabels, oldContainerLabels map[string]string
-		if oldSpec != nil {
-			oldLabels, oldContainerLabels = oldSpec.Labels, oldSpec.Container.Labels
-		}
-		specDiff += formatLabelsDiff("labels:", oldLabels, sp.Spec.Labels, sp.IsNewService)
-		specDiff += formatLabelsDiff("container labels:", oldContainerLabels, sp.Spec.Container.Labels, sp.IsNewService)
+	// Render the labels outside the table because they span multiple lines.
+	if specChanged {
+		specDiff += formatLabelsDiff("labels:", old.Labels, sp.Spec.Labels, sp.IsNewService)
+		specDiff += formatLabelsDiff("container labels:", old.Container.Labels, sp.Spec.Container.Labels,
+			sp.IsNewService)
 	}

 	treePrefix := tui.Faint.Render("  │ ")
@@ -288,6 +300,65 @@ func formatImageDiff(oldImage, newImage string) string {
 		tui.FormatImage(newImage, tui.Green)
 }

+// formatCaddyConfigDiff returns the modifier and the number of added and removed lines in the Caddy config. It omits
+// the config content because it can be large. It returns an empty value if the configs are the same.
+func formatCaddyConfigDiff(oldConfig, newConfig string, isNewService bool) (mod, value string) {
+	if oldConfig == newConfig {
+		return "", ""
+	}
+
+	// Count lines regardless of their order, which is enough for a summary and doesn't require a diff algorithm.
+	remaining := make(map[string]int)
+	var removed int
+	if oldConfig != "" {
+		for line := range strings.SplitSeq(oldConfig, "\n") {
+			remaining[line]++
+			removed++
+		}
+	}
+	var added int
+	if newConfig != "" {
+		for line := range strings.SplitSeq(newConfig, "\n") {
+			if remaining[line] > 0 {
+				remaining[line]--
+				removed--
+			} else {
+				added++
+			}
+		}
+	}
+
+	var parts []string
+	if added > 0 {
+		parts = append(parts, tui.Green.Render("+"+strconv.Itoa(added)))
+	}
+	if removed > 0 {
+		parts = append(parts, tui.Red.Render("-"+strconv.Itoa(removed)))
+	}
+
+	switch {
+	case added+removed == 0:
+		value = "changed"
+	case added+removed == 1:
+		value = parts[0] + " line"
+	default:
+		value = strings.Join(parts, " ") + " lines"
+	}
+
+	switch {
+	case isNewService:
+		mod = ""
+	case oldConfig == "":
+		mod = tui.Green.Render("+")
+	case newConfig == "":
+		mod = tui.Red.Render("-")
+	default:
+		mod = "~"
+	}
+
+	return mod, value
+}
+
 // formatLabelsDiff renders a block with the attribute name followed by a nested line for each added, changed, or
 // removed label, sorted by key. It returns an empty string if the labels are the same.
 func formatLabelsDiff(attribute string, oldLabels, newLabels map[string]string, isNewService bool) string {
@@ -83,6 +83,21 @@ func TestServicePlan_Format(t *testing.T) {
 		}
 	}

+	withCaddy := func(config string) api.ServiceSpec {
+		s := spec("nginx:1.25", nil, nil)
+		s.Caddy = &api.CaddySpec{Config: config}
+		return s
+	}
+	replaceFrom := func(old api.ServiceSpec) func(api.ServiceSpec) []operation.Operation {
+		return func(new api.ServiceSpec) []operation.Operation {
+			return []operation.Operation{&operation.ReplaceContainerOperation{
+				Spec:         new,
+				MachineName:  "machine-1",
+				OldContainer: newContainer(old),
+			}}
+		}
+	}
+
 	tests := []struct {
 		name         string
 		isNewService bool
@@ -90,6 +105,112 @@ func TestServicePlan_Format(t *testing.T) {
 		ops          func(spec api.ServiceSpec) []operation.Operation
 		expected     []string
 	}{
+		{
+			name: "caddy config lines added and removed",
+			spec: withCaddy("{\n\tstorage uncloud\n}"),
+			ops:  replaceFrom(withCaddy("{\n\tdebug\n\tauto_https off\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ ~ caddy config: +1 -2 lines",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config single line added",
+			spec: withCaddy("{\n\tstorage uncloud\n\tdebug\n}"),
+			ops:  replaceFrom(withCaddy("{\n\tstorage uncloud\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ ~ caddy config: +1 line",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config added to existing service",
+			spec: withCaddy("{\n\tstorage uncloud\n}"),
+			ops:  replaceFrom(spec("nginx:1.25", nil, nil)),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ + caddy config: +3 lines",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config removed",
+			spec: spec("nginx:1.25", nil, nil),
+			ops:  replaceFrom(withCaddy("{\n\tstorage uncloud\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ - caddy config: -3 lines",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config with only reordered lines",
+			spec: withCaddy("a {\n}\nb {\n}"),
+			ops:  replaceFrom(withCaddy("b {\n}\na {\n}")),
+			expected: []string{
+				"~ update service web",
+				"  │   image:        nginx:1.25",
+				"  │ ~ caddy config: changed",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name: "caddy config unchanged ignoring surrounding whitespace",
+			spec: withCaddy("{\n\tdebug\n}"),
+			ops:  replaceFrom(withCaddy("\n{\n\tdebug\n}\n")),
+			expected: []string{
+				"~ update service web",
+				"  │   image: nginx:1.25",
+				"  │",
+				"  ╰── +/- replace container web/0f6df6593e1b on machine-1",
+			},
+		},
+		{
+			name:         "new service with caddy config",
+			isNewService: true,
+			spec:         withCaddy("{\n\tstorage uncloud\n}"),
+			ops: func(new api.ServiceSpec) []operation.Operation {
+				return []operation.Operation{&operation.RunContainerOperation{Spec: new, MachineName: "machine-1"}}
+			},
+			expected: []string{
+				"+ create service web",
+				"  │   image:        nginx:1.25",
+				"  │   caddy config: +3 lines",
+				"  │",
+				"  ╰── +   run container web on machine-1",
+			},
+		},
+		{
+			name:         "caddy config before replicas",
+			isNewService: true,
+			spec: func() api.ServiceSpec {
+				s := withCaddy("{\n\tstorage uncloud\n}")
+				s.Replicas = 3
+				return s
+			}(),
+			ops: func(new api.ServiceSpec) []operation.Operation {
+				return []operation.Operation{&operation.RunContainerOperation{Spec: new, MachineName: "machine-1"}}
+			},
+			expected: []string{
+				"+ create service web",
+				"  │   image:        nginx:1.25",
+				"  │   caddy config: +3 lines",
+				"  │   replicas:     3",
+				"  │",
+				"  ╰── +   run container web on machine-1",
+			},
+		},
 		{
 			name: "service labels added, changed, and removed in place",
 			spec: spec("nginx:1.25", map[string]string{"env": "prod", "team": "platform", "same": "1"}, nil),