feat(caddy): make cluster storage for Caddy default, remember 'uc caddy deploy' options

This commit is contained in:
Pasha Sviderski
2026-11-05 16:03:34 +10:00
parent 76b04dd56c
commit 22b1c68798
13 changed files with 1173 additions and 98 deletions
+192 -20
View File
@@ -16,6 +16,7 @@ import (
"github.com/psviderski/uncloud/internal/cli"
"github.com/psviderski/uncloud/internal/cli/completion"
"github.com/psviderski/uncloud/internal/cli/tui"
"github.com/psviderski/uncloud/internal/version"
"github.com/psviderski/uncloud/pkg/api"
"github.com/psviderski/uncloud/pkg/client"
"github.com/spf13/cobra"
@@ -25,6 +26,10 @@ type deployOptions struct {
caddyfile string
image string
machines []string
caddyfileChanged bool
imageChanged bool
machinesChanged bool
}
func NewDeployCommand() *cobra.Command {
@@ -33,21 +38,62 @@ func NewDeployCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "deploy",
Short: "Deploy or upgrade Caddy reverse proxy across all machines in the cluster.",
Long: "Deploy or upgrade Caddy reverse proxy across all machines in the cluster.\n" +
"A rolling update is performed when updating existing containers to minimise disruption.",
Long: `Deploy or upgrade Caddy reverse proxy across all machines in the cluster.
A rolling update is performed when updating existing containers to minimise disruption.
By default, this deploys the latest version of the Uncloud Caddy image with a global config that enables cluster
storage for TLS certificates. See https://github.com/unlabs-dev/caddy-uncloud for more details.
Values set with --image, --caddyfile, and --machine are kept for later deployments. Pass an empty value to reset
a flag to its default.
If this command didn't deploy the current caddy service, for example, it was deployed with 'uc deploy'
from a Compose file or an older version of uc, the deployment replaces its configuration.
Requires Uncloud daemon ` + client.MinCaddyDaemonVersion + ` or newer on all machines.`,
Example: ` # Deploy Caddy or upgrade it to the latest version of the Uncloud Caddy image.
uc caddy deploy
# Deploy a custom image and keep it for later deployments.
uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3
# Go back to the latest version of the Uncloud Caddy image.
uc caddy deploy --image ""
# Deploy a custom image build and a custom global Caddy config and keep both for later deployments.
uc caddy deploy --image myorg/caddy-uncloud:latest --caddyfile global.Caddyfile
# Go back to the default global Caddy config but keep the custom image.
uc caddy deploy --caddyfile ""
# Deploy only to specific machines and keep them for later deployments.
uc caddy deploy --machine machine1,machine2
# Go back to deploying to all machines.
uc caddy deploy --machine ""`,
RunE: func(cmd *cobra.Command, args []string) error {
uncli := cmd.Context().Value("cli").(*cli.CLI)
opts.caddyfileChanged = cmd.Flag("caddyfile").Changed
opts.imageChanged = cmd.Flag("image").Changed
opts.machinesChanged = cmd.Flag("machine").Changed
return runDeploy(cmd.Context(), uncli, opts)
},
}
cmd.Flags().StringVar(&opts.caddyfile, "caddyfile", "",
"Path to a custom global Caddy config (Caddyfile) that will be prepended to the auto-generated Caddy config.")
"Path to a custom global Caddy config (Caddyfile) that will be prepended to the auto-generated Caddy config.\n"+
"An empty value resets it to the default config that enables cluster storage.\n"+
"(default is the current custom config or the default config)")
cmd.Flags().StringVar(&opts.image, "image", "",
"Caddy Docker image to deploy. (default caddy:LATEST_VERSION)")
"Caddy Docker image to deploy. It must include the Uncloud storage module if the Caddy config uses it.\n"+
"An empty value resets it to the latest version of the Uncloud Caddy image.\n"+
"(default is the current custom image or "+client.CaddyImage+":LATEST_VERSION)")
cmd.Flags().StringSliceVarP(&opts.machines, "machine", "m", nil,
"Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated "+
"list. (default is all machines)")
"Machine names or IDs to deploy to. Can be specified multiple times or as a comma-separated list.\n"+
"An empty value resets it to all machines.\n"+
"(default is the current machines or all machines)")
completion.MachinesFlag(cmd)
@@ -55,13 +101,13 @@ func NewDeployCommand() *cobra.Command {
}
func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
caddyfile := ""
config := ""
if opts.caddyfile != "" {
data, err := os.ReadFile(opts.caddyfile)
if err != nil {
return fmt.Errorf("read Caddyfile: %w", err)
}
caddyfile = strings.TrimSpace(string(data))
config = strings.TrimSpace(string(data))
}
clusterClient, err := uncli.ConnectCluster(ctx)
@@ -70,6 +116,10 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
}
defer clusterClient.Close()
if err = clusterClient.Caddy.CheckDaemonVersions(ctx); err != nil {
return err
}
svc, err := clusterClient.InspectService(ctx, client.CaddyServiceName)
if err != nil {
if !errors.Is(err, api.ErrNotFound) {
@@ -100,26 +150,48 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
}
}
if opts.image != "" {
fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(opts.image, tui.Green))
current := svc.Spec()
deployOpts := resolveDeploymentOptions(current, opts, config)
configChanged := deployOpts.ConfigOrDefault() != current.CaddyConfig()
if svc.ID != "" && configChanged {
printCaddyConfig("current global Caddy config:", current.CaddyConfig())
}
fmt.Println()
fmt.Println("Preparing a deployment plan...")
placement := api.Placement{
Machines: cli.ExpandCommaSeparatedValues(opts.machines),
}
d, err := clusterClient.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{
Image: opts.image, Config: caddyfile, Placement: placement,
})
d, err := clusterClient.Caddy.NewDeployment(ctx, deployOpts)
if err != nil {
return fmt.Errorf("create caddy deployment: %w", err)
}
target := d.Spec
if opts.image == "" {
fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(d.Spec.Container.Image,
tui.Green) + tui.Faint.Render(" (latest stable)"))
// The Uncloud Caddy image resolves to the latest version allowed by this CLI, which is older than the running image
// if a newer CLI deployed it. Continuing would downgrade the image and replace its default config.
if deployOpts.Image == "" && client.IsNewerCaddyImage(current.Container.Image, target.Container.Image) {
return fmt.Errorf("caddy service runs image '%s' which is newer than the latest version '%s' supported by "+
"this uc version: upgrade uc or specify the image with --image",
current.Container.Image, target.Container.Image)
}
imageNote := " (latest)"
if deployOpts.Image != "" {
imageNote = " (custom)"
if !opts.imageChanged {
imageNote = " (custom, use --image \"\" for the latest version)"
}
}
fmt.Println(tui.Faint.Render("target image: ") + tui.FormatImage(target.Container.Image, tui.Green) +
tui.Faint.Render(imageNote))
if configChanged {
if deployOpts.Config != nil {
// The user provided this config so there is no need to print its content.
fmt.Println(tui.Faint.Render("target global Caddy config: ") + opts.caddyfile)
} else {
printCaddyConfig("target global Caddy config: (default)", deployOpts.ConfigOrDefault())
}
}
plan, err := d.Plan(ctx)
@@ -131,7 +203,7 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
fmt.Printf("%s service is up to date.\n", client.CaddyServiceName)
} else {
if svc.ID == "" {
if len(opts.machines) > 0 {
if len(deployOpts.Placement.Machines) > 0 {
fmt.Println("This will run a Caddy container on each selected machine.")
} else {
fmt.Println("This will run a Caddy container on each machine.")
@@ -163,6 +235,14 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
fmt.Println(summary)
fmt.Println()
warnings := deploymentWarnings(current, target, opts)
for _, warning := range warnings {
tui.PrintWarning(warning)
}
if len(warnings) > 0 {
fmt.Println()
}
title := "Proceed with deployment?"
// Include the direct connection or context name in the confirmation prompt to avoid accidentally
// deploying to the wrong cluster.
@@ -195,6 +275,98 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
return UpdateDomainRecords(ctx, clusterClient, uncli.ProgressOut())
}
// resolveDeploymentOptions returns the deployment options for the flags and the current service spec, which is zero if
// the service doesn't exist. Each flag works on its own: a specified flag overrides the value and an empty one resets
// it to the default. A flag that isn't specified keeps the explicitly set value.
func resolveDeploymentOptions(
current api.ServiceSpec, opts deployOptions, config string,
) client.CaddyDeploymentOptions {
var resolved client.CaddyDeploymentOptions
if opts.imageChanged {
resolved.Image = opts.image
} else if _, custom := current.Labels[api.LabelCaddyImageCustom]; custom {
resolved.Image = current.Container.Image
}
if opts.caddyfileChanged {
if opts.caddyfile != "" {
resolved.Config = new(config)
}
} else if _, custom := current.Labels[api.LabelCaddyConfigCustom]; custom {
resolved.Config = new(current.CaddyConfig())
}
if opts.machinesChanged {
resolved.Placement.Machines = cli.ExpandCommaSeparatedValues(opts.machines)
} else if _, managed := current.Labels[api.LabelCaddyManaged]; managed {
resolved.Placement = current.Placement
}
return resolved
}
// deploymentWarnings returns the consequences of deploying the target spec over the current one, which is zero if
// the service doesn't exist.
func deploymentWarnings(current, target api.ServiceSpec, opts deployOptions) []string {
var warnings []string
exists := current.Name != ""
if _, managed := current.Labels[api.LabelCaddyManaged]; exists && !managed {
warnings = append(warnings, "'uc caddy deploy' doesn't manage this caddy service yet, so this deployment "+
"replaces its configuration, including any custom global Caddy config.\n"+
"If you manage it with a Compose file, cancel and deploy with 'uc deploy' instead.")
}
currentStorage := client.CaddyStorageModule(current.CaddyConfig())
targetStorage := client.CaddyStorageModule(target.CaddyConfig())
if exists && currentStorage != targetStorage {
warning := fmt.Sprintf("Caddy certificate storage changes from %s to %s.\n"+
"Caddy doesn't automatically migrate certificates, so all of them will be reissued and HTTPS may fail "+
"for those domains until that finishes.\n"+
"To migrate them first, see %s/concepts/ingress/cluster-storage-for-caddy",
caddyStorageName(currentStorage), caddyStorageName(targetStorage), version.DocsURL)
if !opts.caddyfileChanged {
warning += "\nTo keep the current storage, specify a custom Caddyfile that uses it with --caddyfile."
}
warnings = append(warnings, warning)
}
imageOrConfigChanged := current.Container.Image != target.Container.Image ||
current.CaddyConfig() != target.CaddyConfig()
if targetStorage == client.CaddyStorageUncloud && imageOrConfigChanged &&
!client.IsDefaultCaddyImage(target.Container.Image) {
warnings = append(warnings, fmt.Sprintf("Image '%s' must include the Uncloud storage module to use "+
"'storage uncloud'. Otherwise, Caddy won't start or apply the new config.\n"+
"Check 'uc caddy config' after the deployment for errors.", target.Container.Image))
}
return warnings
}
// caddyStorageName returns a user-friendly name of the Caddy storage module.
func caddyStorageName(module string) string {
switch module {
case client.CaddyStorageFileSystem:
return "local"
case client.CaddyStorageUncloud:
return "cluster"
default:
return "'" + module + "'"
}
}
// printCaddyConfig prints the title and the indented config or "none" if it's empty.
func printCaddyConfig(title, config string) {
if config == "" {
lipgloss.Println(tui.Faint.Render(title) + " none")
return
}
lipgloss.Println(tui.Faint.Render(title))
fmt.Println(config)
}
func UpdateDomainRecords(ctx context.Context, clusterClient *client.Client, progressOut *streams.Out) error {
domain, err := clusterClient.GetDomain(ctx)
if err != nil {
+263
View File
@@ -0,0 +1,263 @@
package caddy
import (
"strings"
"testing"
"github.com/psviderski/uncloud/pkg/api"
"github.com/psviderski/uncloud/pkg/client"
"github.com/stretchr/testify/assert"
)
func TestResolveDeploymentOptions(t *testing.T) {
t.Parallel()
const (
customImage = "example.com/caddy:1.2.3"
customConfig = "{\n\tdebug\n}"
)
machines := api.Placement{Machines: []string{"m1", "m2"}}
// A service with a custom image, config, and machines that has the given labels.
service := func(labels ...string) api.ServiceSpec {
spec := api.ServiceSpec{
Name: client.CaddyServiceName,
Caddy: &api.CaddySpec{Config: customConfig},
Container: api.ContainerSpec{Image: customImage},
Placement: machines,
}
if len(labels) > 0 {
spec.Labels = make(map[string]string, len(labels))
for _, label := range labels {
spec.Labels[label] = ""
}
}
return spec
}
tests := []struct {
name string
current api.ServiceSpec
opts deployOptions
config string
want client.CaddyDeploymentOptions
}{
{
name: "no service gets the defaults",
},
{
name: "service that isn't managed gets the defaults",
current: service(),
},
{
name: "managed service follows the default image and config and keeps machines",
current: service(api.LabelCaddyManaged),
want: client.CaddyDeploymentOptions{Placement: machines},
},
{
name: "custom image is kept and config follows the default",
current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom),
want: client.CaddyDeploymentOptions{Image: customImage, Placement: machines},
},
{
name: "custom config is kept and image follows the latest version",
current: service(api.LabelCaddyManaged, api.LabelCaddyConfigCustom),
want: client.CaddyDeploymentOptions{Config: new(customConfig), Placement: machines},
},
{
name: "custom image and config are kept",
current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom, api.LabelCaddyConfigCustom),
want: client.CaddyDeploymentOptions{Image: customImage, Config: new(customConfig), Placement: machines},
},
{
name: "custom empty config is kept",
current: api.ServiceSpec{
Name: client.CaddyServiceName,
Labels: map[string]string{api.LabelCaddyManaged: "", api.LabelCaddyConfigCustom: ""},
},
want: client.CaddyDeploymentOptions{Config: new("")},
},
{
name: "image flag overrides the current image",
current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom),
opts: deployOptions{image: "caddy:2", imageChanged: true},
want: client.CaddyDeploymentOptions{Image: "caddy:2", Placement: machines},
},
{
name: "empty image flag resets to the latest version",
current: service(api.LabelCaddyManaged, api.LabelCaddyImageCustom),
opts: deployOptions{imageChanged: true},
want: client.CaddyDeploymentOptions{Placement: machines},
},
{
name: "caddyfile flag overrides the current config",
current: service(api.LabelCaddyManaged),
opts: deployOptions{caddyfile: "global.Caddyfile", caddyfileChanged: true},
config: "{\n\tstorage redis\n}",
want: client.CaddyDeploymentOptions{Config: new("{\n\tstorage redis\n}"), Placement: machines},
},
{
name: "caddyfile flag with an empty file sets a custom empty config",
current: service(api.LabelCaddyManaged),
opts: deployOptions{caddyfile: "empty.Caddyfile", caddyfileChanged: true},
want: client.CaddyDeploymentOptions{Config: new(""), Placement: machines},
},
{
name: "empty caddyfile flag resets to the default config",
current: service(api.LabelCaddyManaged, api.LabelCaddyConfigCustom),
opts: deployOptions{caddyfileChanged: true},
want: client.CaddyDeploymentOptions{Placement: machines},
},
{
name: "machine flag overrides the current machines",
current: service(api.LabelCaddyManaged),
opts: deployOptions{machines: []string{"m3,m4", "m5"}, machinesChanged: true},
want: client.CaddyDeploymentOptions{Placement: api.Placement{Machines: []string{"m3", "m4", "m5"}}},
},
{
name: "empty machine flag resets to all machines",
current: service(api.LabelCaddyManaged),
opts: deployOptions{machines: []string{}, machinesChanged: true},
},
{
name: "flags apply to a service that isn't managed",
current: service(),
opts: deployOptions{
image: "caddy:2", imageChanged: true,
caddyfile: "global.Caddyfile", caddyfileChanged: true,
machines: []string{"m3"}, machinesChanged: true,
},
config: customConfig,
want: client.CaddyDeploymentOptions{
Image: "caddy:2",
Config: new(customConfig),
Placement: api.Placement{Machines: []string{"m3"}},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, resolveDeploymentOptions(tt.current, tt.opts, tt.config))
})
}
}
func TestDeploymentWarnings(t *testing.T) {
t.Parallel()
const (
defaultImage = client.CaddyImage + ":0.1.3"
customImage = "example.com/caddy:1.2.3"
localConfig = "{\n\tdebug\n}"
override = "doesn't manage this caddy service yet"
storage = "certificate storage changes from"
keepStorage = "To keep the current storage"
module = "must include the Uncloud storage module"
)
managed := map[string]string{api.LabelCaddyManaged: ""}
service := func(image, config string, labels map[string]string) api.ServiceSpec {
spec := api.ServiceSpec{
Name: client.CaddyServiceName,
Container: api.ContainerSpec{Image: image},
Labels: labels,
}
if config != "" {
spec.Caddy = &api.CaddySpec{Config: config}
}
return spec
}
tests := []struct {
name string
current api.ServiceSpec
target api.ServiceSpec
opts deployOptions
want []string
notWant []string
}{
{
name: "new service with defaults has no warnings",
target: service(defaultImage, client.DefaultCaddyConfig, managed),
notWant: []string{override, storage, module},
},
{
name: "new service with a custom image",
target: service(customImage, client.DefaultCaddyConfig, managed),
want: []string{module},
notWant: []string{override, storage},
},
{
name: "service that isn't managed moves to cluster storage",
current: service("caddy:2.11.4", "", nil),
target: service(defaultImage, client.DefaultCaddyConfig, managed),
want: []string{override, "from local to cluster", keepStorage},
notWant: []string{module},
},
{
name: "service that isn't managed already uses cluster storage",
current: service(defaultImage, client.DefaultCaddyConfig, nil),
target: service(defaultImage, client.DefaultCaddyConfig, managed),
want: []string{override},
notWant: []string{storage, module},
},
{
name: "managed service without changes has no warnings",
current: service(defaultImage, client.DefaultCaddyConfig, managed),
target: service(defaultImage, client.DefaultCaddyConfig, managed),
notWant: []string{override, storage, module},
},
{
name: "managed service upgrades the default image",
current: service(client.CaddyImage+":0.1.2", client.DefaultCaddyConfig, managed),
target: service(defaultImage, client.DefaultCaddyConfig, managed),
notWant: []string{override, storage, module},
},
{
name: "custom caddyfile moves to local storage",
current: service(defaultImage, client.DefaultCaddyConfig, managed),
target: service(defaultImage, localConfig, managed),
opts: deployOptions{caddyfile: "global.Caddyfile", caddyfileChanged: true},
want: []string{"from cluster to local"},
notWant: []string{override, keepStorage, module},
},
{
name: "other storage module is named",
current: service(defaultImage, "{\n\tstorage redis\n}", managed),
target: service(defaultImage, client.DefaultCaddyConfig, managed),
want: []string{"from 'redis' to cluster", keepStorage},
},
{
name: "custom image is unchanged with the same config",
current: service(customImage, client.DefaultCaddyConfig, managed),
target: service(customImage, client.DefaultCaddyConfig, managed),
notWant: []string{override, storage, module},
},
{
name: "custom image receives a changed default config",
current: service(customImage, "{\n\tstorage uncloud\n\tdebug\n}", managed),
target: service(customImage, client.DefaultCaddyConfig, managed),
want: []string{module},
notWant: []string{override, storage},
},
{
name: "custom image with local storage",
current: service(defaultImage, localConfig, managed),
target: service(customImage, localConfig, managed),
notWant: []string{override, storage, module},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
warnings := strings.Join(deploymentWarnings(tt.current, tt.target, tt.opts), "\n")
for _, want := range tt.want {
assert.Contains(t, warnings, want)
}
for _, notWant := range tt.notWant {
assert.NotContains(t, warnings, notWant)
}
})
}
}
+4 -22
View File
@@ -187,14 +187,12 @@ func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine,
}
fmt.Println("Machine joined the cluster.")
// TODO: scale the existing Caddy service to the new machine instead of running a new deployment
// that may cause a small downtime.
// Deploy a Caddy service container to the added machine. If caddy service is already deployed on other machines,
// use the deployed image version.
// Deploy the current caddy service spec to run a container on the added machine. This keeps the service
// configuration however it was deployed and shouldn't change the containers on other machines if specs on them
// haven't diverged.
// NOTE: We use the cluster client to inspect and scale the Caddy service because the newly added machine may have
// issues accessing the API of existing machines in the cluster.
// See the issue for more details: https://github.com/psviderski/uncloud/issues/65.
caddyImage := ""
caddySvc, err := clusterClient.InspectService(ctx, client.CaddyServiceName)
if err != nil {
if errors.Is(err, api.ErrNotFound) {
@@ -203,26 +201,10 @@ func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine,
}
return fmt.Errorf("inspect caddy service: %w", err)
}
caddyImage = caddySvc.Containers[0].Container.Config.Image
// Find the latest created container and use its image.
var latestCreated time.Time
for _, c := range caddySvc.Containers[1:] {
created, err := time.Parse(time.RFC3339Nano, c.Container.Created)
if err != nil {
continue
}
if created.After(latestCreated) {
latestCreated = created
caddyImage = c.Container.Config.Image
}
}
fmt.Println()
fmt.Println("Preparing Caddy deployment...")
d, err := clusterClient.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: caddyImage})
if err != nil {
return fmt.Errorf("create caddy deployment: %w", err)
}
d := clusterClient.NewDeployment(caddySvc.Spec(), nil)
plan, err := d.Plan(ctx)
if err != nil {