diff --git a/README.md b/README.md index 76665154..6d522238 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,8 @@ complexity of Kubernetes. [Uncloud DNS](https://github.com/psviderski/uncloud-dns) service. * **Automatic HTTPS**: Built-in Caddy reverse proxy handles TLS certificate provisioning and renewal using Let's Encrypt. -* **Cluster storage for TLS certificates**: Native cluster storage for Caddy shares certificates and ACME challenge - tokens across machines and coordinates issuance. +* **Cluster storage for TLS certificates**: [Cluster storage](https://github.com/unlabs-dev/caddy-uncloud) for Caddy + shares certificates and ACME challenge tokens across machines and coordinates issuance. * **Docker-like CLI**: Familiar commands for managing both infrastructure and applications. * **Remote management**: Control your entire infrastructure through SSH access to any single machine in the cluster. diff --git a/cmd/uc/caddy/deploy.go b/cmd/uc/caddy/deploy.go index 357399b6..3bbb7dfe 100644 --- a/cmd/uc/caddy/deploy.go +++ b/cmd/uc/caddy/deploy.go @@ -16,6 +16,7 @@ import ( "github.com/psviderski/uncloud/internal/cli" "github.com/psviderski/uncloud/internal/cli/completion" "github.com/psviderski/uncloud/internal/cli/tui" + "github.com/psviderski/uncloud/internal/version" "github.com/psviderski/uncloud/pkg/api" "github.com/psviderski/uncloud/pkg/client" "github.com/spf13/cobra" @@ -25,6 +26,10 @@ type deployOptions struct { caddyfile string image string machines []string + + caddyfileChanged bool + imageChanged bool + machinesChanged bool } func NewDeployCommand() *cobra.Command { @@ -33,21 +38,62 @@ func NewDeployCommand() *cobra.Command { cmd := &cobra.Command{ Use: "deploy", Short: "Deploy or upgrade Caddy reverse proxy across all machines in the cluster.", - Long: "Deploy or upgrade Caddy reverse proxy across all machines in the cluster.\n" + - "A rolling update is performed when updating existing containers to minimise disruption.", + Long: `Deploy or upgrade Caddy reverse proxy across all machines in the cluster. +A rolling update is performed when updating existing containers to minimise disruption. + +By default, this deploys the latest version of the Uncloud Caddy image with a global config that enables cluster +storage for TLS certificates. See https://github.com/unlabs-dev/caddy-uncloud for more details. + +Values set with --image, --caddyfile, and --machine are kept for later deployments. Pass an empty value to reset +a flag to its default. + +If this command didn't deploy the current caddy service, for example, it was deployed with 'uc deploy' +from a Compose file or an older version of uc, the deployment replaces its configuration. + +Requires Uncloud daemon ` + client.MinCaddyDaemonVersion + ` or newer on all machines.`, + Example: ` # Deploy Caddy or upgrade it to the latest version of the Uncloud Caddy image. + uc caddy deploy + + # Deploy a custom image and keep it for later deployments. + uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3 + + # Go back to the latest version of the Uncloud Caddy image. + uc caddy deploy --image "" + + # Deploy a custom image build and a custom global Caddy config and keep both for later deployments. + uc caddy deploy --image myorg/caddy-uncloud:latest --caddyfile global.Caddyfile + + # Go back to the default global Caddy config but keep the custom image. + uc caddy deploy --caddyfile "" + + # Deploy only to specific machines and keep them for later deployments. + uc caddy deploy --machine machine1,machine2 + + # Go back to deploying to all machines. + uc caddy deploy --machine ""`, RunE: func(cmd *cobra.Command, args []string) error { uncli := cmd.Context().Value("cli").(*cli.CLI) + + opts.caddyfileChanged = cmd.Flag("caddyfile").Changed + opts.imageChanged = cmd.Flag("image").Changed + opts.machinesChanged = cmd.Flag("machine").Changed + return runDeploy(cmd.Context(), uncli, opts) }, } cmd.Flags().StringVar(&opts.caddyfile, "caddyfile", "", - "Path to a custom global Caddy config (Caddyfile) that will be prepended to the auto-generated Caddy config.") + "Path to a custom global Caddy config (Caddyfile) that will be prepended to the auto-generated Caddy config.\n"+ + "An empty value resets it to the default config that enables cluster storage.\n"+ + "(default is the current custom config or the default config)") cmd.Flags().StringVar(&opts.image, "image", "", - "Caddy Docker image to deploy. (default caddy:LATEST_VERSION)") + "Caddy Docker image to deploy. It must include the Uncloud storage module if the Caddy config uses it.\n"+ + "An empty value resets it to the latest version of the Uncloud Caddy image.\n"+ + "(default is the current custom image or "+client.CaddyImage+":LATEST_VERSION)") cmd.Flags().StringSliceVarP(&opts.machines, "machine", "m", nil, - "Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated "+ - "list. (default is all machines)") + "Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated list.\n"+ + "An empty value resets it to all machines.\n"+ + "(default is the current machines or all machines)") completion.MachinesFlag(cmd) @@ -55,13 +101,13 @@ func NewDeployCommand() *cobra.Command { } func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error { - caddyfile := "" + config := "" if opts.caddyfile != "" { data, err := os.ReadFile(opts.caddyfile) if err != nil { return fmt.Errorf("read Caddyfile: %w", err) } - caddyfile = strings.TrimSpace(string(data)) + config = strings.TrimSpace(string(data)) } clusterClient, err := uncli.ConnectCluster(ctx) @@ -70,6 +116,10 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error { } defer clusterClient.Close() + if err = clusterClient.Caddy.CheckDaemonVersions(ctx); err != nil { + return err + } + svc, err := clusterClient.InspectService(ctx, client.CaddyServiceName) if err != nil { if !errors.Is(err, api.ErrNotFound) { @@ -100,26 +150,48 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error { } } - if opts.image != "" { - fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(opts.image, tui.Green)) + current := svc.Spec() + deployOpts := resolveDeploymentOptions(current, opts, config) + configChanged := deployOpts.ConfigOrDefault() != current.CaddyConfig() + + if svc.ID != "" && configChanged { + printCaddyConfig("current global Caddy config:", current.CaddyConfig()) } fmt.Println() fmt.Println("Preparing a deployment plan...") - placement := api.Placement{ - Machines: cli.ExpandCommaSeparatedValues(opts.machines), - } - d, err := clusterClient.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{ - Image: opts.image, Config: caddyfile, Placement: placement, - }) + d, err := clusterClient.Caddy.NewDeployment(ctx, deployOpts) if err != nil { return fmt.Errorf("create caddy deployment: %w", err) } + target := d.Spec - if opts.image == "" { - fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(d.Spec.Container.Image, - tui.Green) + tui.Faint.Render(" (latest stable)")) + // The Uncloud Caddy image resolves to the latest version allowed by this CLI, which is older than the running image + // if a newer CLI deployed it. Continuing would downgrade the image and replace its default config. + if deployOpts.Image == "" && client.IsNewerCaddyImage(current.Container.Image, target.Container.Image) { + return fmt.Errorf("caddy service runs image '%s' which is newer than the latest version '%s' supported by "+ + "this uc version: upgrade uc or specify the image with --image", + current.Container.Image, target.Container.Image) + } + + imageNote := " (latest)" + if deployOpts.Image != "" { + imageNote = " (custom)" + if !opts.imageChanged { + imageNote = " (custom, use --image \"\" for the latest version)" + } + } + fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(target.Container.Image, tui.Green) + + tui.Faint.Render(imageNote)) + + if configChanged { + if deployOpts.Config != nil { + // The user provided this config so there is no need to print its content. + fmt.Println(tui.Faint.Render("target global Caddy config: ") + opts.caddyfile) + } else { + printCaddyConfig("target global Caddy config: (default)", deployOpts.ConfigOrDefault()) + } } plan, err := d.Plan(ctx) @@ -131,7 +203,7 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error { fmt.Printf("%s service is up to date.\n", client.CaddyServiceName) } else { if svc.ID == "" { - if len(opts.machines) > 0 { + if len(deployOpts.Placement.Machines) > 0 { fmt.Println("This will run a Caddy container on each selected machine.") } else { fmt.Println("This will run a Caddy container on each machine.") @@ -163,6 +235,14 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error { fmt.Println(summary) fmt.Println() + warnings := deploymentWarnings(current, target, opts) + for _, warning := range warnings { + tui.PrintWarning(warning) + } + if len(warnings) > 0 { + fmt.Println() + } + title := "Proceed with deployment?" // Include the direct connection or context name in the confirmation prompt to avoid accidentally // deploying to the wrong cluster. @@ -195,6 +275,98 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error { return UpdateDomainRecords(ctx, clusterClient, uncli.ProgressOut()) } +// resolveDeploymentOptions returns the deployment options for the flags and the current service spec, which is zero if +// the service doesn't exist. Each flag works on its own: a specified flag overrides the value and an empty one resets +// it to the default. A flag that isn't specified keeps the explicitly set value. +func resolveDeploymentOptions( + current api.ServiceSpec, opts deployOptions, config string, +) client.CaddyDeploymentOptions { + var resolved client.CaddyDeploymentOptions + + if opts.imageChanged { + resolved.Image = opts.image + } else if _, custom := current.Labels[api.LabelCaddyImageCustom]; custom { + resolved.Image = current.Container.Image + } + + if opts.caddyfileChanged { + if opts.caddyfile != "" { + resolved.Config = new(config) + } + } else if _, custom := current.Labels[api.LabelCaddyConfigCustom]; custom { + resolved.Config = new(current.CaddyConfig()) + } + + if opts.machinesChanged { + resolved.Placement.Machines = cli.ExpandCommaSeparatedValues(opts.machines) + } else if _, managed := current.Labels[api.LabelCaddyManaged]; managed { + resolved.Placement = current.Placement + } + + return resolved +} + +// deploymentWarnings returns the consequences of deploying the target spec over the current one, which is zero if +// the service doesn't exist. +func deploymentWarnings(current, target api.ServiceSpec, opts deployOptions) []string { + var warnings []string + exists := current.Name != "" + + if _, managed := current.Labels[api.LabelCaddyManaged]; exists && !managed { + warnings = append(warnings, "'uc caddy deploy' doesn't manage this caddy service yet, so this deployment "+ + "replaces its configuration, including any custom global Caddy config.\n"+ + "If you manage it with a Compose file, cancel and deploy with 'uc deploy' instead.") + } + + currentStorage := client.CaddyStorageModule(current.CaddyConfig()) + targetStorage := client.CaddyStorageModule(target.CaddyConfig()) + if exists && currentStorage != targetStorage { + warning := fmt.Sprintf("Caddy certificate storage changes from %s to %s.\n"+ + "Caddy doesn't automatically migrate certificates, so all of them will be reissued and HTTPS may fail "+ + "for those domains until that finishes.\n"+ + "To migrate them first, see %s/concepts/ingress/cluster-storage-for-caddy", + caddyStorageName(currentStorage), caddyStorageName(targetStorage), version.DocsURL) + if !opts.caddyfileChanged { + warning += "\nTo keep the current storage, specify a custom Caddyfile that uses it with --caddyfile." + } + warnings = append(warnings, warning) + } + + imageOrConfigChanged := current.Container.Image != target.Container.Image || + current.CaddyConfig() != target.CaddyConfig() + if targetStorage == client.CaddyStorageUncloud && imageOrConfigChanged && + !client.IsDefaultCaddyImage(target.Container.Image) { + warnings = append(warnings, fmt.Sprintf("Image '%s' must include the Uncloud storage module to use "+ + "'storage uncloud'. Otherwise, Caddy won't start or apply the new config.\n"+ + "Check 'uc caddy config' after the deployment for errors.", target.Container.Image)) + } + + return warnings +} + +// caddyStorageName returns a user-friendly name of the Caddy storage module. +func caddyStorageName(module string) string { + switch module { + case client.CaddyStorageFileSystem: + return "local" + case client.CaddyStorageUncloud: + return "cluster" + default: + return "'" + module + "'" + } +} + +// printCaddyConfig prints the title and the indented config or "none" if it's empty. +func printCaddyConfig(title, config string) { + if config == "" { + lipgloss.Println(tui.Faint.Render(title) + " none") + return + } + + lipgloss.Println(tui.Faint.Render(title)) + fmt.Println(config) +} + func UpdateDomainRecords(ctx context.Context, clusterClient *client.Client, progressOut *streams.Out) error { domain, err := clusterClient.GetDomain(ctx) if err != nil { diff --git a/cmd/uc/caddy/deploy_test.go b/cmd/uc/caddy/deploy_test.go new file mode 100644 index 00000000..7491a64c --- /dev/null +++ b/cmd/uc/caddy/deploy_test.go @@ -0,0 +1,263 @@ +package caddy + +import ( + "strings" + "testing" + + "github.com/psviderski/uncloud/pkg/api" + "github.com/psviderski/uncloud/pkg/client" + "github.com/stretchr/testify/assert" +) + +func TestResolveDeploymentOptions(t *testing.T) { + t.Parallel() + + const ( + customImage = "example.com/caddy:1.2.3" + customConfig = "{\n\tdebug\n}" + ) + machines := api.Placement{Machines: []string{"m1", "m2"}} + + // A service with a custom image, config, and machines that has the given labels. + service := func(labels ...string) api.ServiceSpec { + spec := api.ServiceSpec{ + Name: client.CaddyServiceName, + Caddy: &api.CaddySpec{Config: customConfig}, + Container: api.ContainerSpec{Image: customImage}, + Placement: machines, + } + if len(labels) > 0 { + spec.Labels = make(map[string]string, len(labels)) + for _, label := range labels { + spec.Labels[label] = "" + } + } + return spec + } + + tests := []struct { + name string + current api.ServiceSpec + opts deployOptions + config string + want client.CaddyDeploymentOptions + }{ + { + name: "no service gets the defaults", + }, + { + name: "service that isn't managed gets the defaults", + current: service(), + }, + { + name: "managed service follows the default image and config and keeps machines", + current: service(api.LabelCaddyManaged), + want: client.CaddyDeploymentOptions{Placement: machines}, + }, + { + name: "custom image is kept and config follows the default", + current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom), + want: client.CaddyDeploymentOptions{Image: customImage, Placement: machines}, + }, + { + name: "custom config is kept and image follows the latest version", + current: service(api.LabelCaddyManaged, api.LabelCaddyConfigCustom), + want: client.CaddyDeploymentOptions{Config: new(customConfig), Placement: machines}, + }, + { + name: "custom image and config are kept", + current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom, api.LabelCaddyConfigCustom), + want: client.CaddyDeploymentOptions{Image: customImage, Config: new(customConfig), Placement: machines}, + }, + { + name: "custom empty config is kept", + current: api.ServiceSpec{ + Name: client.CaddyServiceName, + Labels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""}, + }, + want: client.CaddyDeploymentOptions{Config: new("")}, + }, + { + name: "image flag overrides the current image", + current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom), + opts: deployOptions{image: "caddy:2", imageChanged: true}, + want: client.CaddyDeploymentOptions{Image: "caddy:2", Placement: machines}, + }, + { + name: "empty image flag resets to the latest version", + current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom), + opts: deployOptions{imageChanged: true}, + want: client.CaddyDeploymentOptions{Placement: machines}, + }, + { + name: "caddyfile flag overrides the current config", + current: service(api.LabelCaddyManaged), + opts: deployOptions{caddyfile: "global.Caddyfile", caddyfileChanged: true}, + config: "{\n\tstorage redis\n}", + want: client.CaddyDeploymentOptions{Config: new("{\n\tstorage redis\n}"), Placement: machines}, + }, + { + name: "caddyfile flag with an empty file sets a custom empty config", + current: service(api.LabelCaddyManaged), + opts: deployOptions{caddyfile: "empty.Caddyfile", caddyfileChanged: true}, + want: client.CaddyDeploymentOptions{Config: new(""), Placement: machines}, + }, + { + name: "empty caddyfile flag resets to the default config", + current: service(api.LabelCaddyManaged, api.LabelCaddyConfigCustom), + opts: deployOptions{caddyfileChanged: true}, + want: client.CaddyDeploymentOptions{Placement: machines}, + }, + { + name: "machine flag overrides the current machines", + current: service(api.LabelCaddyManaged), + opts: deployOptions{machines: []string{"m3,m4", "m5"}, machinesChanged: true}, + want: client.CaddyDeploymentOptions{Placement: api.Placement{Machines: []string{"m3", "m4", "m5"}}}, + }, + { + name: "empty machine flag resets to all machines", + current: service(api.LabelCaddyManaged), + opts: deployOptions{machines: []string{}, machinesChanged: true}, + }, + { + name: "flags apply to a service that isn't managed", + current: service(), + opts: deployOptions{ + image: "caddy:2", imageChanged: true, + caddyfile: "global.Caddyfile", caddyfileChanged: true, + machines: []string{"m3"}, machinesChanged: true, + }, + config: customConfig, + want: client.CaddyDeploymentOptions{ + Image: "caddy:2", + Config: new(customConfig), + Placement: api.Placement{Machines: []string{"m3"}}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, resolveDeploymentOptions(tt.current, tt.opts, tt.config)) + }) + } +} + +func TestDeploymentWarnings(t *testing.T) { + t.Parallel() + + const ( + defaultImage = client.CaddyImage + ":0.1.3" + customImage = "example.com/caddy:1.2.3" + localConfig = "{\n\tdebug\n}" + override = "doesn't manage this caddy service yet" + storage = "certificate storage changes from" + keepStorage = "To keep the current storage" + module = "must include the Uncloud storage module" + ) + managed := map[string]string{api.LabelCaddyManaged: ""} + + service := func(image, config string, labels map[string]string) api.ServiceSpec { + spec := api.ServiceSpec{ + Name: client.CaddyServiceName, + Container: api.ContainerSpec{Image: image}, + Labels: labels, + } + if config != "" { + spec.Caddy = &api.CaddySpec{Config: config} + } + return spec + } + + tests := []struct { + name string + current api.ServiceSpec + target api.ServiceSpec + opts deployOptions + want []string + notWant []string + }{ + { + name: "new service with defaults has no warnings", + target: service(defaultImage, client.DefaultCaddyConfig, managed), + notWant: []string{override, storage, module}, + }, + { + name: "new service with a custom image", + target: service(customImage, client.DefaultCaddyConfig, managed), + want: []string{module}, + notWant: []string{override, storage}, + }, + { + name: "service that isn't managed moves to cluster storage", + current: service("caddy:2.11.4", "", nil), + target: service(defaultImage, client.DefaultCaddyConfig, managed), + want: []string{override, "from local to cluster", keepStorage}, + notWant: []string{module}, + }, + { + name: "service that isn't managed already uses cluster storage", + current: service(defaultImage, client.DefaultCaddyConfig, nil), + target: service(defaultImage, client.DefaultCaddyConfig, managed), + want: []string{override}, + notWant: []string{storage, module}, + }, + { + name: "managed service without changes has no warnings", + current: service(defaultImage, client.DefaultCaddyConfig, managed), + target: service(defaultImage, client.DefaultCaddyConfig, managed), + notWant: []string{override, storage, module}, + }, + { + name: "managed service upgrades the default image", + current: service(client.CaddyImage+":0.1.2", client.DefaultCaddyConfig, managed), + target: service(defaultImage, client.DefaultCaddyConfig, managed), + notWant: []string{override, storage, module}, + }, + { + name: "custom caddyfile moves to local storage", + current: service(defaultImage, client.DefaultCaddyConfig, managed), + target: service(defaultImage, localConfig, managed), + opts: deployOptions{caddyfile: "global.Caddyfile", caddyfileChanged: true}, + want: []string{"from cluster to local"}, + notWant: []string{override, keepStorage, module}, + }, + { + name: "other storage module is named", + current: service(defaultImage, "{\n\tstorage redis\n}", managed), + target: service(defaultImage, client.DefaultCaddyConfig, managed), + want: []string{"from 'redis' to cluster", keepStorage}, + }, + { + name: "custom image is unchanged with the same config", + current: service(customImage, client.DefaultCaddyConfig, managed), + target: service(customImage, client.DefaultCaddyConfig, managed), + notWant: []string{override, storage, module}, + }, + { + name: "custom image receives a changed default config", + current: service(customImage, "{\n\tstorage uncloud\n\tdebug\n}", managed), + target: service(customImage, client.DefaultCaddyConfig, managed), + want: []string{module}, + notWant: []string{override, storage}, + }, + { + name: "custom image with local storage", + current: service(defaultImage, localConfig, managed), + target: service(customImage, localConfig, managed), + notWant: []string{override, storage, module}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + warnings := strings.Join(deploymentWarnings(tt.current, tt.target, tt.opts), "\n") + for _, want := range tt.want { + assert.Contains(t, warnings, want) + } + for _, notWant := range tt.notWant { + assert.NotContains(t, warnings, notWant) + } + }) + } +} diff --git a/cmd/uc/machine/add.go b/cmd/uc/machine/add.go index 577f00d7..90c6f2de 100644 --- a/cmd/uc/machine/add.go +++ b/cmd/uc/machine/add.go @@ -187,14 +187,12 @@ func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine, } fmt.Println("Machine joined the cluster.") - // TODO: scale the existing Caddy service to the new machine instead of running a new deployment - // that may cause a small downtime. - // Deploy a Caddy service container to the added machine. If caddy service is already deployed on other machines, - // use the deployed image version. + // Deploy the current caddy service spec to run a container on the added machine. This keeps the service + // configuration however it was deployed and shouldn't change the containers on other machines if specs on them + // haven't diverged. // NOTE: We use the cluster client to inspect and scale the Caddy service because the newly added machine may have // issues accessing the API of existing machines in the cluster. // See the issue for more details: https://github.com/psviderski/uncloud/issues/65. - caddyImage := "" caddySvc, err := clusterClient.InspectService(ctx, client.CaddyServiceName) if err != nil { if errors.Is(err, api.ErrNotFound) { @@ -203,26 +201,10 @@ func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine, } return fmt.Errorf("inspect caddy service: %w", err) } - caddyImage = caddySvc.Containers[0].Container.Config.Image - // Find the latest created container and use its image. - var latestCreated time.Time - for _, c := range caddySvc.Containers[1:] { - created, err := time.Parse(time.RFC3339Nano, c.Container.Created) - if err != nil { - continue - } - if created.After(latestCreated) { - latestCreated = created - caddyImage = c.Container.Config.Image - } - } fmt.Println() fmt.Println("Preparing Caddy deployment...") - d, err := clusterClient.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: caddyImage}) - if err != nil { - return fmt.Errorf("create caddy deployment: %w", err) - } + d := clusterClient.NewDeployment(caddySvc.Spec(), nil) plan, err := d.Plan(ctx) if err != nil { diff --git a/pkg/api/caddy.go b/pkg/api/caddy.go index ba5ad57b..91a82f26 100644 --- a/pkg/api/caddy.go +++ b/pkg/api/caddy.go @@ -5,6 +5,15 @@ import ( "time" ) +const ( + // LabelCaddyManaged marks the caddy service as managed by 'uc caddy deploy'. + LabelCaddyManaged = LabelReservedPrefix + "caddy.managed" + // LabelCaddyImageCustom marks the caddy image as explicitly set instead of following the latest default image. + LabelCaddyImageCustom = LabelReservedPrefix + "caddy.image.custom" + // LabelCaddyConfigCustom marks the global Caddy config as explicitly set instead of following the default config. + LabelCaddyConfigCustom = LabelReservedPrefix + "caddy.config.custom" +) + // CaddyConfig is the saved Caddy configuration on a machine. type CaddyConfig struct { Caddyfile string diff --git a/pkg/client/caddy.go b/pkg/client/caddy.go index cfbe2480..1228866a 100644 --- a/pkg/client/caddy.go +++ b/pkg/client/caddy.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "regexp" + "strings" "time" "github.com/Masterminds/semver" @@ -11,6 +12,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/psviderski/uncloud/api/pb" + "github.com/psviderski/uncloud/internal/version" "github.com/psviderski/uncloud/pkg/api" "github.com/psviderski/uncloud/pkg/client/deploy" "google.golang.org/protobuf/types/known/emptypb" @@ -18,11 +20,33 @@ import ( const ( CaddyServiceName = "caddy" - // CaddyImage is the official Caddy Docker image on Docker Hub: https://hub.docker.com/_/caddy - CaddyImage = "caddy" + // CaddyImage is the Uncloud Caddy image that is deployed by default. It includes the Uncloud storage module for + // sharing TLS certificates across machines: https://github.com/unlabs-dev/caddy-uncloud + CaddyImage = "ghcr.io/unlabs-dev/caddy-uncloud" + // DefaultCaddyConfig is the default global Caddyfile. It must be compatible with all versions of CaddyImage allowed + // by caddyImageConstraint. + DefaultCaddyConfig = "{\n\tstorage uncloud\n}" + // CaddyStorageFileSystem is the name of the default Caddy storage module that stores TLS assets locally. + CaddyStorageFileSystem = "file_system" + // CaddyStorageUncloud is the name of the Caddy storage module that stores TLS assets in the cluster. + CaddyStorageUncloud = "uncloud" + // MinCaddyDaemonVersion is the minimum daemon version required for 'uc caddy deploy'. Older daemons don't store + // service labels that mark the service as managed by this command. Distributed locks were introduced in 0.21.0, + // which is the minimum required by the cluster storage even on machines that don't run the Caddy service. + MinCaddyDaemonVersion = "0.22.0" + + // officialCaddyImage is the official Caddy Docker image on Docker Hub: https://hub.docker.com/_/caddy + officialCaddyImage = "docker.io/library/caddy" ) -var caddyImageTagRegex = regexp.MustCompile(`^2\.\d+\.\d+$`) +var ( + // caddyImageConstraint limits CaddyImage to the major version that is known to be compatible with + // DefaultCaddyConfig. + caddyImageConstraint = mustNewConstraint("<1.0.0") + // caddyImageTagRegex matches the exact version tags of CaddyImage, skipping floating tags and image variants. + caddyImageTagRegex = regexp.MustCompile(`^\d+\.\d+\.\d+$`) + caddyStorageRegex = regexp.MustCompile(`(?m)^[ \t]*storage[ \t]+(\S+)`) +) // CaddyClient provides Caddy operations over its parent Client's connection. // The parent client owns the connection and must be used to close it. @@ -64,19 +88,28 @@ func (c *CaddyClient) Config(ctx context.Context, opts CaddyConfigOptions) (api. return config, nil } -// CaddyDeploymentOptions configures a Caddy reverse proxy deployment. +// CaddyDeploymentOptions configures a Caddy reverse proxy deployment. The zero value deploys the latest version of +// CaddyImage with DefaultCaddyConfig to all machines. type CaddyDeploymentOptions struct { - // Image defaults to the latest stable 2.x.x official Caddy image. - // Custom images must include curl and start Caddy with /etc/caddy/Caddyfile. + // Image defaults to the latest version of CaddyImage. A non-empty value is marked as custom. + // Custom images must include curl and start Caddy with /etc/caddy/Caddyfile. They must also include the Uncloud + // storage module if the config uses 'storage uncloud'. Image string - // Config contains an optional global Caddyfile. - Config string + // Config is the global Caddyfile. Nil means DefaultCaddyConfig. Any other value, even empty, is marked as custom. + Config *string Placement api.Placement } -// NewDeployment creates a new deployment for a Caddy reverse proxy service. -// The service is deployed in global mode to all machines in the cluster. If the image is not provided, the latest -// version of the official Caddy Docker image is used. +// ConfigOrDefault returns the global Caddyfile of the deployment. +func (o CaddyDeploymentOptions) ConfigOrDefault() string { + if o.Config != nil { + return *o.Config + } + return DefaultCaddyConfig +} + +// NewDeployment creates a new managed deployment for a Caddy reverse proxy service. +// The service is deployed in global mode to all machines in the cluster or the machines set in the placement. func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOptions) (*deploy.Deployment, error) { if err := ctx.Err(); err != nil { return nil, err @@ -90,6 +123,10 @@ func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOpt image = reference.FamiliarString(latest) } + config := opts.ConfigOrDefault() + if err := validateCaddyImageStorage(image, config); err != nil { + return nil, err + } spec := api.ServiceSpec{ Container: api.ContainerSpec{ @@ -186,17 +223,119 @@ func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOpt }, } - if opts.Config != "" { + if config != "" { spec.Caddy = &api.CaddySpec{ - Config: opts.Config, + Config: config, } } + spec.Labels = map[string]string{api.LabelCaddyManaged: ""} + if opts.Image != "" { + spec.Labels[api.LabelCaddyImageCustom] = "" + } + if opts.Config != nil { + spec.Labels[api.LabelCaddyConfigCustom] = "" + } + return c.client.NewDeployment(spec, nil), nil } -// latestCaddyImage returns the latest image of the official Caddy Docker image on Docker Hub. -// The latest image is determined by the latest version tag 2.x.x. +// CheckDaemonVersions returns an error naming the available machines whose daemons are older than +// MinCaddyDaemonVersion. Machines that are down are skipped because a deployment doesn't touch them. +func (c *CaddyClient) CheckDaemonVersions(ctx context.Context) error { + machines, err := c.client.ListMachines(ctx, &api.MachineFilter{Available: true}) + if err != nil { + return fmt.Errorf("list machines: %w", err) + } + + if outdated := outdatedCaddyDaemons(machines); len(outdated) > 0 { + return fmt.Errorf("deploying Caddy requires Uncloud daemon %s or newer on all machines, upgrade these "+ + "machines and try again: %s. See https://github.com/psviderski/uncloud/releases/latest", + MinCaddyDaemonVersion, strings.Join(outdated, ", ")) + } + return nil +} + +// outdatedCaddyDaemons returns descriptions of the machines with outdated daemons. Prereleases and development builds +// aren't considered outdated because their versions don't show whether they're older than a stable release. +func outdatedCaddyDaemons(machines api.MachineMembersList) []string { + minVersion := semver.MustParse(MinCaddyDaemonVersion) + + var outdated []string + for _, m := range machines { + v, err := semver.NewVersion(m.Machine.DaemonVersion) + if err != nil || v.Prerelease() != "" { + continue + } + if v.LessThan(minVersion) { + outdated = append(outdated, fmt.Sprintf("%s (%s)", m.Machine.Name, m.Machine.DaemonVersion)) + } + } + + return outdated +} + +// CaddyStorageModule returns the name of the storage module set in the global options of a Caddyfile. +func CaddyStorageModule(config string) string { + if match := caddyStorageRegex.FindStringSubmatch(config); match != nil { + return match[1] + } + return CaddyStorageFileSystem +} + +// IsDefaultCaddyImage returns whether the image is any version of CaddyImage. +func IsDefaultCaddyImage(image string) bool { + return imageName(image) == CaddyImage +} + +// IsNewerCaddyImage returns whether the image is a newer version of CaddyImage than the other image. +// It returns false if any of the images isn't a version of CaddyImage. +func IsNewerCaddyImage(image, other string) bool { + version, otherVersion := caddyImageVersion(image), caddyImageVersion(other) + return version != nil && otherVersion != nil && version.GreaterThan(otherVersion) +} + +// caddyImageVersion returns the version in the image tag if the image is a version of CaddyImage, or nil otherwise. +func caddyImageVersion(image string) *semver.Version { + named, err := reference.ParseNormalizedNamed(image) + if err != nil || named.Name() != CaddyImage { + return nil + } + tagged, ok := named.(reference.Tagged) + if !ok { + return nil + } + version, err := semver.NewVersion(tagged.Tag()) + if err != nil { + return nil + } + + return version +} + +// imageName returns the normalised name of the image without its tag or digest. +// It returns an empty string if the image reference is invalid. +func imageName(image string) string { + named, err := reference.ParseNormalizedNamed(image) + if err != nil { + return "" + } + return named.Name() +} + +// validateCaddyImageStorage rejects the official Caddy image with a config that uses the Uncloud storage module +// because it doesn't include this module. Otherwise, a new container could start from the previously saved Caddyfile +// and pass its health check, but its routes would stop updating as Caddy would reject the new config. +func validateCaddyImageStorage(image, config string) error { + if imageName(image) == officialCaddyImage && CaddyStorageModule(config) == CaddyStorageUncloud { + return fmt.Errorf("official Caddy image '%s' doesn't support 'storage uncloud' in the global Caddy config: "+ + "use an image that includes the Uncloud storage module or a Caddyfile without this option. "+ + "See %s/concepts/ingress/cluster-storage-for-caddy", image, version.DocsURL) + } + return nil +} + +// latestCaddyImage returns the latest version of CaddyImage allowed by caddyImageConstraint. func latestCaddyImage(ctx context.Context) (reference.NamedTagged, error) { if err := ctx.Err(); err != nil { return nil, err @@ -222,7 +361,7 @@ func latestCaddyImage(ctx context.Context) (reference.NamedTagged, error) { return imageWithTag, nil } -// latestCaddyTag selects the newest stable 2.x.x tag, falling back to latest. +// latestCaddyTag selects the newest tag matching caddyImageTagRegex and caddyImageConstraint, falling back to latest. func latestCaddyTag(tags []string) string { latestTag := "latest" var latestVersion *semver.Version @@ -232,7 +371,7 @@ func latestCaddyTag(tags []string) string { } v, err := semver.NewVersion(t) - if err != nil { + if err != nil || !caddyImageConstraint.Check(v) { continue } if latestVersion == nil || v.GreaterThan(latestVersion) { @@ -243,3 +382,12 @@ func latestCaddyTag(tags []string) string { return latestTag } + +// mustNewConstraint parses a version constraint and panics if it's invalid. +func mustNewConstraint(constraint string) *semver.Constraints { + c, err := semver.NewConstraint(constraint) + if err != nil { + panic(fmt.Sprintf("invalid version constraint '%s': %v", constraint, err)) + } + return c +} diff --git a/pkg/client/caddy_test.go b/pkg/client/caddy_test.go index 3b3eccd8..6ec3a109 100644 --- a/pkg/client/caddy_test.go +++ b/pkg/client/caddy_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/distribution/reference" + "github.com/psviderski/uncloud/api/pb" + "github.com/psviderski/uncloud/pkg/api" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -15,7 +17,7 @@ func TestLatestCaddyImage(t *testing.T) { image, err := latestCaddyImage(context.Background()) require.NoError(t, err) - assert.Regexp(t, `^caddy:2\.\d+\.\d+$`, reference.FamiliarString(image)) + assert.Regexp(t, `^ghcr\.io/unlabs-dev/caddy-uncloud:0\.\d+\.\d+$`, reference.FamiliarString(image)) } func TestLatestCaddyTag(t *testing.T) { @@ -27,14 +29,209 @@ func TestLatestCaddyTag(t *testing.T) { want string }{ {name: "empty", want: "latest"}, - {name: "semantic ordering", tags: []string{"2.9.9", "2.11.4", "2.10.0", "2.11.3"}, want: "2.11.4"}, - {name: "ignore other versions and variants", tags: []string{ - "latest", "1.0.0", "3.0.0", "2.12.0-rc.1", "2.12.0-alpine", "2.12", "v2.12.0", "2.11.4", - }, want: "2.11.4"}, - {name: "no stable version", tags: []string{"builder", "2.12.0-beta.1"}, want: "latest"}, + {name: "semantic ordering", tags: []string{"0.1.9", "0.1.11", "0.1.10", "0.1.2"}, want: "0.1.11"}, + {name: "newest minor version", tags: []string{"0.1.3", "0.2.0", "0.0.9"}, want: "0.2.0"}, + {name: "ignore newer major versions", tags: []string{"1.0.0", "0.1.3", "2.3.4", "1.0.1"}, want: "0.1.3"}, + {name: "ignore variants", tags: []string{ + "latest", "0.1.4-rc.1", "0.1.4-alpine", "0.1", "v0.1.4", "1.0.0-rc.1", "0.1.3", + }, want: "0.1.3"}, + {name: "no stable version", tags: []string{"builder", "0.1.4-beta.1"}, want: "latest"}, } { t.Run(tt.name, func(t *testing.T) { assert.Equal(t, tt.want, latestCaddyTag(tt.tags)) }) } } + +func TestDefaultCaddyConfig_UsesUncloudStorage(t *testing.T) { + t.Parallel() + + assert.Equal(t, CaddyStorageUncloud, CaddyStorageModule(DefaultCaddyConfig)) +} + +func TestCaddyStorageModule(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + config string + want string + }{ + {name: "empty config", want: CaddyStorageFileSystem}, + {name: "no storage option", config: "{\n\tdebug\n}", want: CaddyStorageFileSystem}, + {name: "uncloud", config: "{\n\tstorage uncloud\n}", want: CaddyStorageUncloud}, + {name: "uncloud with options", config: "{\n\tstorage uncloud {\n\t\tfoo bar\n\t}\n}", want: CaddyStorageUncloud}, + {name: "spaces around", config: "{\n storage uncloud \n}", want: CaddyStorageUncloud}, + {name: "other module", config: "{\n\tstorage redis\n}", want: "redis"}, + {name: "explicit file system", config: "{\n\tstorage file_system /data\n}", want: CaddyStorageFileSystem}, + {name: "commented out", config: "{\n\t# storage uncloud\n}", want: CaddyStorageFileSystem}, + {name: "similar option", config: "{\n\tstorage_clean_interval 24h\n}", want: CaddyStorageFileSystem}, + {name: "no module on the same line", config: "{\n\tstorage\n\tdebug\n}", want: CaddyStorageFileSystem}, + } { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, CaddyStorageModule(tt.config)) + }) + } +} + +func TestIsDefaultCaddyImage(t *testing.T) { + t.Parallel() + + assert.True(t, IsDefaultCaddyImage("ghcr.io/unlabs-dev/caddy-uncloud")) + assert.True(t, IsDefaultCaddyImage("ghcr.io/unlabs-dev/caddy-uncloud:0.1.3")) + assert.False(t, IsDefaultCaddyImage("caddy:2.11.4")) + assert.False(t, IsDefaultCaddyImage("example.com/unlabs-dev/caddy-uncloud:0.1.3")) + assert.False(t, IsDefaultCaddyImage("")) +} + +func TestIsNewerCaddyImage(t *testing.T) { + t.Parallel() + + const repo = "ghcr.io/unlabs-dev/caddy-uncloud" + for _, tt := range []struct { + name string + image string + other string + want bool + }{ + {name: "newer major version", image: repo + ":1.0.0", other: repo + ":0.1.3", want: true}, + {name: "newer minor version", image: repo + ":0.2.0", other: repo + ":0.1.3", want: true}, + {name: "same", image: repo + ":0.1.3", other: repo + ":0.1.3"}, + {name: "older", image: repo + ":0.1.2", other: repo + ":0.1.3"}, + {name: "different repo", image: "example.com/custom:1.0.0", other: repo + ":0.1.3"}, + {name: "official image", image: "caddy:2.11.4", other: repo + ":0.1.3"}, + {name: "tag without version", image: repo + ":latest", other: repo + ":0.1.3"}, + {name: "no tag", image: repo, other: repo + ":0.1.3"}, + {name: "fallback to latest tag", image: repo + ":1.0.0", other: repo + ":latest"}, + {name: "no current image", other: repo + ":0.1.3"}, + } { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, IsNewerCaddyImage(tt.image, tt.other)) + }) + } +} + +func TestCaddyClientNewDeployment(t *testing.T) { + t.Parallel() + + const customImage = "example.com/caddy:1.2.3" + for _, tt := range []struct { + name string + opts CaddyDeploymentOptions + wantConfig string + wantLabels map[string]string + }{ + { + name: "defaults", + wantConfig: DefaultCaddyConfig, + wantLabels: map[string]string{api.LabelCaddyManaged: ""}, + }, + { + name: "custom image", + opts: CaddyDeploymentOptions{Image: customImage}, + wantConfig: DefaultCaddyConfig, + wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyImageCustom: ""}, + }, + { + name: "custom config", + opts: CaddyDeploymentOptions{Config: new("{\n\tdebug\n}")}, + wantConfig: "{\n\tdebug\n}", + wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""}, + }, + { + name: "custom empty config", + opts: CaddyDeploymentOptions{Config: new("")}, + wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""}, + }, + { + name: "custom config matching the default is still custom", + opts: CaddyDeploymentOptions{Config: new(DefaultCaddyConfig)}, + wantConfig: DefaultCaddyConfig, + wantLabels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""}, + }, + { + name: "custom image and config", + opts: CaddyDeploymentOptions{ + Image: customImage, + Config: new("{\n\tdebug\n}"), + }, + wantConfig: "{\n\tdebug\n}", + wantLabels: map[string]string{ + api.LabelCaddyManaged: "", + api.LabelCaddyImageCustom: "", + api.LabelCaddyConfigCustom: "", + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + c := &CaddyClient{client: &Client{}} + d, err := c.NewDeployment(context.Background(), tt.opts) + require.NoError(t, err) + + assert.Equal(t, tt.wantConfig, d.Spec.CaddyConfig()) + assert.Equal(t, tt.wantLabels, d.Spec.Labels) + }) + } + + t.Run("official image needs a config without cluster storage", func(t *testing.T) { + c := &CaddyClient{client: &Client{}} + _, err := c.NewDeployment(context.Background(), CaddyDeploymentOptions{Image: "caddy:2.11.4"}) + require.ErrorContains(t, err, "doesn't support 'storage uncloud'") + + _, err = c.NewDeployment(context.Background(), CaddyDeploymentOptions{Image: "caddy:2.11.4", Config: new("")}) + require.NoError(t, err) + }) +} + +func TestValidateCaddyImageStorage(t *testing.T) { + t.Parallel() + + const storageConfig = "{\n\tstorage uncloud\n}" + for _, tt := range []struct { + name string + image string + config string + wantErr bool + }{ + {name: "official image with uncloud storage", image: "caddy:2.11.4", config: storageConfig, wantErr: true}, + {name: "official image with registry", image: "docker.io/library/caddy:2", config: storageConfig, wantErr: true}, + {name: "official image without config", image: "caddy:2.11.4"}, + {name: "official image with other config", image: "caddy:2.11.4", config: "{\n\tdebug\n}"}, + {name: "default image with uncloud storage", image: CaddyImage + ":0.1.3", config: storageConfig}, + {name: "custom image with uncloud storage", image: "example.com/caddy:1", config: storageConfig}, + } { + t.Run(tt.name, func(t *testing.T) { + err := validateCaddyImageStorage(tt.image, tt.config) + if tt.wantErr { + require.ErrorContains(t, err, "doesn't support 'storage uncloud'") + } else { + require.NoError(t, err) + } + }) + } +} + +func TestOutdatedCaddyDaemons(t *testing.T) { + t.Parallel() + + machine := func(name, version string) *pb.MachineMember { + return &pb.MachineMember{Machine: &pb.MachineInfo{Name: name, DaemonVersion: version}} + } + + assert.Equal(t, []string{"old (0.21.0)", "older (0.20.0)", "unsupported (0.9.3)"}, + outdatedCaddyDaemons(api.MachineMembersList{ + machine("old", "0.21.0"), + machine("older", "0.20.0"), + machine("unsupported", "0.9.3"), + machine("no-version", ""), + machine("required", "0.22.0"), + machine("newer", "0.23.1"), + machine("major", "1.0.0"), + machine("nightly", "0.22.0-nightly-abc1234"), + machine("old-dev-build", "v0.21.0-5-g6c7ddeb-dirty"), + machine("local-build", "999.0.0-dev"), + machine("not-semver", "6c7ddeb"), + })) + assert.Empty(t, outdatedCaddyDaemons(api.MachineMembersList{machine("required", "0.22.0")})) + assert.Empty(t, outdatedCaddyDaemons(nil)) +} diff --git a/pkg/client/deploy/deploy.go b/pkg/client/deploy/deploy.go index 12b93280..2e0bca8e 100644 --- a/pkg/client/deploy/deploy.go +++ b/pkg/client/deploy/deploy.go @@ -134,6 +134,22 @@ func (sp *ServicePlan) Format() string { specTable.Row(mod, "image:", formatImageDiff(oldSpec.Container.Image, sp.Spec.Container.Image)) } + // An existing service without an old spec only has new containers added to up-to-date ones so its Caddy config and + // labels didn't change. + specChanged := sp.IsNewService || oldSpec != nil + var old api.ServiceSpec + if oldSpec != nil { + old = *oldSpec + } + + // Caddy config row. + if specChanged { + mod, value := formatCaddyConfigDiff(old.CaddyConfig(), sp.Spec.CaddyConfig(), sp.IsNewService) + if value != "" { + specTable.Row(mod, "caddy config:", value) + } + } + // Replicas row for replicated services. if sp.Spec.Mode == api.ServiceModeReplicated { replicasStr := fmt.Sprintf("%d", sp.Spec.Replicas) @@ -153,15 +169,11 @@ func (sp *ServicePlan) Format() string { specDiff := specTable.String() + "\n" - // Render labels as nested blocks outside the table to not widen its attribute column. An existing service without - // an old spec only has new containers added to up-to-date ones so its labels didn't change. - if sp.IsNewService || oldSpec != nil { - var oldLabels, oldContainerLabels map[string]string - if oldSpec != nil { - oldLabels, oldContainerLabels = oldSpec.Labels, oldSpec.Container.Labels - } - specDiff += formatLabelsDiff("labels:", oldLabels, sp.Spec.Labels, sp.IsNewService) - specDiff += formatLabelsDiff("container labels:", oldContainerLabels, sp.Spec.Container.Labels, sp.IsNewService) + // Render the labels outside the table because they span multiple lines. + if specChanged { + specDiff += formatLabelsDiff("labels:", old.Labels, sp.Spec.Labels, sp.IsNewService) + specDiff += formatLabelsDiff("container labels:", old.Container.Labels, sp.Spec.Container.Labels, + sp.IsNewService) } treePrefix := tui.Faint.Render(" │ ") @@ -288,6 +300,65 @@ func formatImageDiff(oldImage, newImage string) string { tui.FormatImage(newImage, tui.Green) } +// formatCaddyConfigDiff returns the modifier and the number of added and removed lines in the Caddy config. It omits +// the config content because it can be large. It returns an empty value if the configs are the same. +func formatCaddyConfigDiff(oldConfig, newConfig string, isNewService bool) (mod, value string) { + if oldConfig == newConfig { + return "", "" + } + + // Count lines regardless of their order, which is enough for a summary and doesn't require a diff algorithm. + remaining := make(map[string]int) + var removed int + if oldConfig != "" { + for line := range strings.SplitSeq(oldConfig, "\n") { + remaining[line]++ + removed++ + } + } + var added int + if newConfig != "" { + for line := range strings.SplitSeq(newConfig, "\n") { + if remaining[line] > 0 { + remaining[line]-- + removed-- + } else { + added++ + } + } + } + + var parts []string + if added > 0 { + parts = append(parts, tui.Green.Render("+"+strconv.Itoa(added))) + } + if removed > 0 { + parts = append(parts, tui.Red.Render("-"+strconv.Itoa(removed))) + } + + switch { + case added+removed == 0: + value = "changed" + case added+removed == 1: + value = parts[0] + " line" + default: + value = strings.Join(parts, " ") + " lines" + } + + switch { + case isNewService: + mod = "" + case oldConfig == "": + mod = tui.Green.Render("+") + case newConfig == "": + mod = tui.Red.Render("-") + default: + mod = "~" + } + + return mod, value +} + // formatLabelsDiff renders a block with the attribute name followed by a nested line for each added, changed, or // removed label, sorted by key. It returns an empty string if the labels are the same. func formatLabelsDiff(attribute string, oldLabels, newLabels map[string]string, isNewService bool) string { diff --git a/pkg/client/deploy/deploy_test.go b/pkg/client/deploy/deploy_test.go index d8010b56..2d81b8cf 100644 --- a/pkg/client/deploy/deploy_test.go +++ b/pkg/client/deploy/deploy_test.go @@ -83,6 +83,21 @@ func TestServicePlan_Format(t *testing.T) { } } + withCaddy := func(config string) api.ServiceSpec { + s := spec("nginx:1.25", nil, nil) + s.Caddy = &api.CaddySpec{Config: config} + return s + } + replaceFrom := func(old api.ServiceSpec) func(api.ServiceSpec) []operation.Operation { + return func(new api.ServiceSpec) []operation.Operation { + return []operation.Operation{&operation.ReplaceContainerOperation{ + Spec: new, + MachineName: "machine-1", + OldContainer: newContainer(old), + }} + } + } + tests := []struct { name string isNewService bool @@ -90,6 +105,112 @@ func TestServicePlan_Format(t *testing.T) { ops func(spec api.ServiceSpec) []operation.Operation expected []string }{ + { + name: "caddy config lines added and removed", + spec: withCaddy("{\n\tstorage uncloud\n}"), + ops: replaceFrom(withCaddy("{\n\tdebug\n\tauto_https off\n}")), + expected: []string{ + "~ update service web", + " │ image: nginx:1.25", + " │ ~ caddy config: +1 -2 lines", + " │", + " ╰── +/- replace container web/0f6df6593e1b on machine-1", + }, + }, + { + name: "caddy config single line added", + spec: withCaddy("{\n\tstorage uncloud\n\tdebug\n}"), + ops: replaceFrom(withCaddy("{\n\tstorage uncloud\n}")), + expected: []string{ + "~ update service web", + " │ image: nginx:1.25", + " │ ~ caddy config: +1 line", + " │", + " ╰── +/- replace container web/0f6df6593e1b on machine-1", + }, + }, + { + name: "caddy config added to existing service", + spec: withCaddy("{\n\tstorage uncloud\n}"), + ops: replaceFrom(spec("nginx:1.25", nil, nil)), + expected: []string{ + "~ update service web", + " │ image: nginx:1.25", + " │ + caddy config: +3 lines", + " │", + " ╰── +/- replace container web/0f6df6593e1b on machine-1", + }, + }, + { + name: "caddy config removed", + spec: spec("nginx:1.25", nil, nil), + ops: replaceFrom(withCaddy("{\n\tstorage uncloud\n}")), + expected: []string{ + "~ update service web", + " │ image: nginx:1.25", + " │ - caddy config: -3 lines", + " │", + " ╰── +/- replace container web/0f6df6593e1b on machine-1", + }, + }, + { + name: "caddy config with only reordered lines", + spec: withCaddy("a {\n}\nb {\n}"), + ops: replaceFrom(withCaddy("b {\n}\na {\n}")), + expected: []string{ + "~ update service web", + " │ image: nginx:1.25", + " │ ~ caddy config: changed", + " │", + " ╰── +/- replace container web/0f6df6593e1b on machine-1", + }, + }, + { + name: "caddy config unchanged ignoring surrounding whitespace", + spec: withCaddy("{\n\tdebug\n}"), + ops: replaceFrom(withCaddy("\n{\n\tdebug\n}\n")), + expected: []string{ + "~ update service web", + " │ image: nginx:1.25", + " │", + " ╰── +/- replace container web/0f6df6593e1b on machine-1", + }, + }, + { + name: "new service with caddy config", + isNewService: true, + spec: withCaddy("{\n\tstorage uncloud\n}"), + ops: func(new api.ServiceSpec) []operation.Operation { + return []operation.Operation{&operation.RunContainerOperation{Spec: new, MachineName: "machine-1"}} + }, + expected: []string{ + "+ create service web", + " │ image: nginx:1.25", + " │ caddy config: +3 lines", + " │", + " ╰── + run container web on machine-1", + }, + }, + { + name: "caddy config before replicas", + isNewService: true, + spec: func() api.ServiceSpec { + s := withCaddy("{\n\tstorage uncloud\n}") + s.Replicas = 3 + return s + }(), + ops: func(new api.ServiceSpec) []operation.Operation { + return []operation.Operation{&operation.RunContainerOperation{Spec: new, MachineName: "machine-1"}} + }, + expected: []string{ + "+ create service web", + " │ image: nginx:1.25", + " │ caddy config: +3 lines", + " │ replicas: 3", + " │", + " ╰── + run container web on machine-1", + }, + }, { name: "service labels added, changed, and removed in place", spec: spec("nginx:1.25", map[string]string{"env": "prod", "team": "platform", "same": "1"}, nil), diff --git a/test/e2e/service_test.go b/test/e2e/service_test.go index a86b3b16..7e0e53fa 100644 --- a/test/e2e/service_test.go +++ b/test/e2e/service_test.go @@ -319,6 +319,8 @@ func TestDeployment(t *testing.T) { deployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{}) require.NoError(t, err) deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod + image := deployment.Spec.Container.Image + assert.Regexp(t, `^ghcr\.io/unlabs-dev/caddy-uncloud:0\.\d+\.\d+$`, image) _, err = deployment.Run(ctx) require.NoError(t, err) @@ -327,15 +329,72 @@ func TestDeployment(t *testing.T) { require.NoError(t, err) assert.Len(t, svc.Containers, 3) assertServiceMatchesSpec(t, svc, deployment.Spec) + assert.Equal(t, map[string]string{api.LabelCaddyManaged: ""}, svc.Spec().Labels) - ctr := svc.Containers[0].Container - assert.Regexp(t, `^caddy:2\.\d+\.\d+$`, ctr.Config.Image) - - config, err := cli.Caddy.Config(ctx, client.CaddyConfigOptions{}) - require.NoError(t, err) - + var config api.CaddyConfig + require.Eventually(t, func() bool { + config, err = cli.Caddy.Config(ctx, client.CaddyConfigOptions{}) + return err == nil && strings.Contains(config.Caddyfile, "storage uncloud") + }, 10*time.Second, 200*time.Millisecond, "expected the Caddyfile to enable cluster storage") + assert.Empty(t, config.LastReconciliationError) assert.Contains(t, config.Caddyfile, "# Caddyfile autogenerated by Uncloud") assert.Contains(t, config.Caddyfile, "handle /.uncloud-verify") + + // Setting the running image as custom only changes the reserved service labels so it must not recreate + // the containers. + containerIDs := serviceContainerIDs(svc) + deployment, err = cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: image}) + require.NoError(t, err) + + _, err = deployment.Run(ctx) + require.NoError(t, err) + + svc, err = cli.InspectService(ctx, client.CaddyServiceName) + require.NoError(t, err) + assert.True(t, containerIDs.Equal(serviceContainerIDs(svc)), "expected the containers to be updated in place") + assert.Equal(t, map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyImageCustom: ""}, svc.Spec().Labels) + }) + + t.Run("caddy current spec deployment only adds a container on a new machine", func(t *testing.T) { + t.Cleanup(func() { + err := cli.RemoveService(ctx, client.CaddyServiceName) + if !errors.Is(err, api.ErrNotFound) { + require.NoError(t, err) + } + }) + + deployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{}) + require.NoError(t, err) + deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod + + _, err = deployment.Run(ctx) + require.NoError(t, err) + + svc, err := cli.InspectService(ctx, client.CaddyServiceName) + require.NoError(t, err) + require.Len(t, svc.Containers, 3) + + // Remove a container to make its machine look like a new one that doesn't run Caddy yet, as 'uc machine add' sees it. + removed := svc.Containers[0] + err = cli.RemoveContainer(ctx, svc.ID, removed.Container.ID, container.RemoveOptions{Force: true}) + require.NoError(t, err) + + svc, err = cli.InspectService(ctx, client.CaddyServiceName) + require.NoError(t, err) + require.Len(t, svc.Containers, 2) + containerIDs := serviceContainerIDs(svc) + + _, err = cli.NewDeployment(svc.Spec(), nil).Run(ctx) + require.NoError(t, err) + + svc, err = cli.InspectService(ctx, client.CaddyServiceName) + require.NoError(t, err) + assert.Len(t, svc.Containers, 3) + assert.Len(t, serviceMachines(svc).ToSlice(), 3, "expected 1 container on each machine") + assert.True(t, containerIDs.IsSubset(serviceContainerIDs(svc)), + "expected other containers to be left unchanged") + assertServiceMatchesSpec(t, svc, deployment.Spec) + assert.Equal(t, map[string]string{api.LabelCaddyManaged: ""}, svc.Spec().Labels) }) t.Run("caddy with machine placement", func(t *testing.T) { @@ -365,6 +424,11 @@ func TestDeployment(t *testing.T) { assert.Equal(t, c.Machines[0].ID, svc.Containers[0].MachineID) // initialContainerID := svc.Containers[0].Container.ID + // Deploying the current spec keeps the placement so it must not add containers on other machines. + plan, err := cli.NewDeployment(svc.Spec(), nil).Plan(ctx) + require.NoError(t, err) + assert.Empty(t, plan.Operations) + // Deploy to all machines without a placement constraint. deployment, err = cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: image}) require.NoError(t, err) @@ -447,7 +511,7 @@ func TestDeployment(t *testing.T) { myapp.example.com { reverse_proxy 1.2.3.4:8000 }` - caddyDeployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Config: caddyCaddyfile}) + caddyDeployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Config: &caddyCaddyfile}) require.NoError(t, err) caddyDeployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod @@ -1832,11 +1896,21 @@ func TestServiceLifecycle(t *testing.T) { require.NoError(t, err) assertContainerMatchesSpec(t, mc.Container, spec) - // Reserved service labels are rejected. + // Service labels with an invalid key format are rejected. err = cli.UpdateContainer(ctx, serviceID, ctr.ID, api.UpdateContainerOptions{ - ServiceLabels: &map[string]string{api.LabelManaged: ""}, + ServiceLabels: &map[string]string{"my label": ""}, }) - require.ErrorContains(t, err, "reserved prefix") + require.ErrorContains(t, err, "must not contain whitespace") + + // The daemon accepts reserved service labels because Uncloud sets its own labels on services. + err = cli.UpdateContainer(ctx, serviceID, ctr.ID, api.UpdateContainerOptions{ + ServiceLabels: &map[string]string{"uncloud.future": ""}, + }) + require.NoError(t, err) + + mc, err = cli.InspectContainer(ctx, serviceID, ctr.ID) + require.NoError(t, err) + assert.Equal(t, map[string]string{"uncloud.future": ""}, mc.Container.ServiceSpec.Labels) require.NoError(t, cli.RemoveContainer(ctx, serviceID, ctr.ID, container.RemoveOptions{Force: true})) err = cli.UpdateContainer(ctx, serviceID, ctr.ID, api.UpdateContainerOptions{ diff --git a/website/docs/2-getting-started/2-deploy-demo-app.md b/website/docs/2-getting-started/2-deploy-demo-app.md index 3517865c..bf6b23f7 100644 --- a/website/docs/2-getting-started/2-deploy-demo-app.md +++ b/website/docs/2-getting-started/2-deploy-demo-app.md @@ -156,7 +156,7 @@ Waiting for the machine to be ready... Reserved cluster domain: sh8hsb.uncld.dev [+] Deploying service caddy 2/2 ✔ Container caddy-d7uk on machine-dc3c Running 11.1s - ✔ Image caddy:2.11.4 on machine-dc3c Pulled 3.7s + ✔ Image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3 on machine-dc3c Pulled 3.7s Updating cluster domain records in Uncloud DNS to point to machines running caddy service... [+] Verifying internet access to caddy service 1/1 @@ -216,9 +216,9 @@ uc ls ``` ``` -NAME MODE REPLICAS IMAGE ENDPOINTS -caddy global 1 caddy:2.11.4 -excalidraw replicated 1 excalidraw/excalidraw:latest https://excalidraw.sh8hsb.uncld.dev → :80 +NAME MODE REPLICAS IMAGE ENDPOINTS +caddy global 1 ghcr.io/unlabs-dev/caddy-uncloud:0.1.3 +excalidraw replicated 1 excalidraw/excalidraw:latest https://excalidraw.sh8hsb.uncld.dev → :80 ``` You can see `caddy` service listed here. That's your reverse proxy, running as a regular Uncloud service. diff --git a/website/docs/3-concepts/2-ingress/1-overview.md b/website/docs/3-concepts/2-ingress/1-overview.md index 16e1978e..78397eaa 100644 --- a/website/docs/3-concepts/2-ingress/1-overview.md +++ b/website/docs/3-concepts/2-ingress/1-overview.md @@ -25,10 +25,6 @@ When you [publish a service port](2-publishing-services.md), Uncloud automatical For advanced use cases, Uncloud allows to customise the Caddy config using the `x-caddy` extension in Compose files. See [Custom Caddy configuration](2-publishing-services.md#custom-caddy-configuration) for details. -:::tip Shared TLS certificates - -Uncloud also provides native cluster storage for Caddy. It shares TLS certificates and ACME challenge tokens across +Uncloud configures Caddy to use cluster storage by default. It shares TLS certificates and ACME challenge tokens across machines and coordinates certificate issuance. See [Cluster storage for Caddy](4-cluster-storage-for-caddy.md) -to enable it. - -::: +for details. diff --git a/website/docs/9-cli-reference/uc_caddy_deploy.md b/website/docs/9-cli-reference/uc_caddy_deploy.md index f6c26edc..eea2adf7 100644 --- a/website/docs/9-cli-reference/uc_caddy_deploy.md +++ b/website/docs/9-cli-reference/uc_caddy_deploy.md @@ -7,17 +7,59 @@ Deploy or upgrade Caddy reverse proxy across all machines in the cluster. Deploy or upgrade Caddy reverse proxy across all machines in the cluster. A rolling update is performed when updating existing containers to minimise disruption. +By default, this deploys the latest version of the Uncloud Caddy image with a global config that enables cluster +storage for TLS certificates. See https://github.com/unlabs-dev/caddy-uncloud for more details. + +Values set with --image, --caddyfile, and --machine are kept for later deployments. Pass an empty value to reset +a flag to its default. + +If this command didn't deploy the current caddy service, for example, it was deployed with 'uc deploy' +from a Compose file or an older version of uc, the deployment replaces its configuration. + +Requires Uncloud daemon 0.22.0 or newer on all machines. + ``` uc caddy deploy [flags] ``` +## Examples + +``` + # Deploy Caddy or upgrade it to the latest version of the Uncloud Caddy image. + uc caddy deploy + + # Deploy a custom image and keep it for later deployments. + uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3 + + # Go back to the latest version of the Uncloud Caddy image. + uc caddy deploy --image "" + + # Deploy a custom image build and a custom global Caddy config and keep both for later deployments. + uc caddy deploy --image myorg/caddy-uncloud:latest --caddyfile global.Caddyfile + + # Go back to the default global Caddy config but keep the custom image. + uc caddy deploy --caddyfile "" + + # Deploy only to specific machines and keep them for later deployments. + uc caddy deploy --machine machine1,machine2 + + # Go back to deploying to all machines. + uc caddy deploy --machine "" +``` + ## Options ``` --caddyfile string Path to a custom global Caddy config (Caddyfile) that will be prepended to the auto-generated Caddy config. + An empty value resets it to the default config that enables cluster storage. + (default is the current custom config or the default config) -h, --help help for deploy - --image string Caddy Docker image to deploy. (default caddy:LATEST_VERSION) - -m, --machine strings Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated list. (default is all machines) + --image string Caddy Docker image to deploy. It must include the Uncloud storage module if the Caddy config uses it. + An empty value resets it to the latest version of the Uncloud Caddy image. + (default is the current custom image or ghcr.io/unlabs-dev/caddy-uncloud:LATEST_VERSION) + -m, --machine strings Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated list. + An empty value resets it to all machines. + (default is the current machines or all machines) ``` ## Options inherited from parent commands