feat: add --recreate flag for deploy command to force container recreation

README.md

@@ -323,6 +323,15 @@ SQLite database used to share Uncloud's cluster state.
   features, and be the first to know when it's ready for production use.
 * Watch this repository for releases.
 
+## 💖 Sponsors
+
+These companies and projects are helping Uncloud with their generous sponsorship and/or services:
+
+<!-- Sentry -->
+<a href="https://sentry.io/welcome/">
+  <img height="100" alt="Sentry" src="https://github.com/user-attachments/assets/6c1439c0-d20d-40dc-a669-c9aa94651dfa" />
+</a>
+
 ## ❤️ Contributors
 
 Thank you [@cedws](https://github.com/cedws) for being the first contributor to Uncloud! 🎉

cmd/uncloud/deploy.go

@@ -21,6 +21,7 @@ type deployOptions struct {
 	profiles []string
 	services []string
 	noBuild  bool
+	recreate bool
 
 	context string
 }
@@ -50,6 +51,8 @@ func NewDeployCommand() *cobra.Command {
 		"Name of the cluster context to deploy to (default is the current context)")
 	cmd.Flags().BoolVarP(&opts.noBuild, "no-build", "n", false,
 		"Do not build images before deploying services. (default false)")
+	cmd.Flags().BoolVar(&opts.recreate, "recreate", false,
+		"Recreate containers even if their configuration and image haven't changed.")
 
 	// TODO: Consider adding a filter flag to specify which machines to deploy to but keep the rest running.
 	//  Could be useful to test a new version on a subset of machines before rolling out to all.
@@ -108,7 +111,11 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 	}
 	defer clusterClient.Close()
 
-	composeDeploy, err := compose.NewDeployment(ctx, clusterClient, project)
+	var strategy deploy.Strategy
+	if opts.recreate {
+		strategy = &deploy.RollingStrategy{ForceRecreate: true}
+	}
+	composeDeploy, err := compose.NewDeploymentWithStrategy(ctx, clusterClient, project, strategy)
 	if err != nil {
 		return fmt.Errorf("create compose deployment: %w", err)
 	}

internal/machine/docker/server.go

@@ -546,10 +546,10 @@ func (s *Server) CreateServiceContainer(
 			Memory:            spec.Container.Resources.Memory,
 			MemoryReservation: spec.Container.Resources.MemoryReservation,
 		},
-		// Always restart service containers if they exit or a machine restarts.
+		// Restart service containers if they exit or a machine restarts unless they are explicitly stopped.
 		// For one-off containers and batch jobs we plan to use a different service type/mode.
 		RestartPolicy: container.RestartPolicy{
-			Name: container.RestartPolicyAlways,
+			Name: container.RestartPolicyUnlessStopped,
 		},
 	}
 

pkg/client/compose/deploy.go

@@ -25,11 +25,16 @@ type Deployment struct {
 	Client       Client
 	Project      *types.Project
 	SpecResolver *deploy.ServiceSpecResolver
+	Strategy     deploy.Strategy
 	state        *scheduler.ClusterState
 	plan         *deploy.SequenceOperation
 }
 
 func NewDeployment(ctx context.Context, cli Client, project *types.Project) (*Deployment, error) {
+	return NewDeploymentWithStrategy(ctx, cli, project, nil)
+}
+
+func NewDeploymentWithStrategy(ctx context.Context, cli Client, project *types.Project, strategy deploy.Strategy) (*Deployment, error) {
 	state, err := scheduler.InspectClusterState(ctx, cli)
 	if err != nil {
 		return nil, fmt.Errorf("inspect cluster state: %w", err)
@@ -39,16 +44,20 @@ func NewDeployment(ctx context.Context, cli Client, project *types.Project) (*De
 	if err != nil && !errors.Is(err, api.ErrNotFound) {
 		return nil, fmt.Errorf("get cluster domain: %w", err)
 	}
-
 	resolver := &deploy.ServiceSpecResolver{
 		// If the domain is not found (not reserved), an empty domain is used for the resolver.
 		ClusterDomain: domain,
 	}
 
+	if strategy == nil {
+		strategy = &deploy.RollingStrategy{State: state}
+	}
+
 	return &Deployment{
 		Client:       cli,
 		Project:      project,
 		SpecResolver: resolver,
+		Strategy:     strategy,
 		state:        state,
 	}, nil
 }
@@ -90,7 +99,7 @@ func (d *Deployment) Plan(ctx context.Context) (deploy.SequenceOperation, error)
 	for _, spec := range serviceSpecs {
 		// TODO: properly handle depends_on conditions in the service deployment plan as the first operation.
 		// Pass the update cluster state with scheduled volumes to the deployment.
-		deployment := deploy.NewDeployment(d.Client, spec, &deploy.RollingStrategy{State: d.state})
+		deployment := deploy.NewDeployment(d.Client, spec, d.Strategy)
 		servicePlan, err := deployment.Plan(ctx)
 		if err != nil {
 			return plan, fmt.Errorf("create deployment plan for service '%s': %w", spec.Name, err)

pkg/client/deploy/strategy.go

@@ -26,6 +26,7 @@ type Strategy interface {
 // to minimize service disruption.
 type RollingStrategy struct {
 	State         *scheduler.ClusterState
+	ForceRecreate bool
 }
 
 func (s *RollingStrategy) Type() string {
@@ -92,7 +93,12 @@ func (s *RollingStrategy) planReplicated(svc *api.Service, spec api.ServiceSpec)
 				continue
 			}
 
-			status := EvalContainerSpecChange(c.Container.ServiceSpec, spec)
+			var status ContainerSpecStatus
+			if s.ForceRecreate {
+				status = ContainerNeedsRecreate
+			} else {
+				status = EvalContainerSpecChange(c.Container.ServiceSpec, spec)
+			}
 			containerSpecStatuses[c.Container.ID] = status
 
 			if status == ContainerUpToDate {
@@ -225,7 +231,7 @@ func (s *RollingStrategy) planGlobal(svc *api.Service, spec api.ServiceSpec) (Pl
 
 	for _, m := range availableMachines {
 		containers := containersOnMachine[m.Info.Id]
-		ops, err := reconcileGlobalContainer(containers, spec, plan.ServiceID, m.Info.Id)
+		ops, err := reconcileGlobalContainer(containers, spec, plan.ServiceID, m.Info.Id, s.ForceRecreate)
 		if err != nil {
 			return plan, err
 		}
@@ -252,7 +258,7 @@ func (s *RollingStrategy) planGlobal(svc *api.Service, spec api.ServiceSpec) (Pl
 // It ensures exactly one container with the desired spec is running on the machine by creating a new container and
 // removing old ones. If there is a host port conflict, it stops the old container before starting a new one.
 func reconcileGlobalContainer(
-	containers []api.MachineServiceContainer, spec api.ServiceSpec, serviceID, machineID string,
+	containers []api.MachineServiceContainer, spec api.ServiceSpec, serviceID, machineID string, forceRecreate bool,
 ) ([]Operation, error) {
 	var ops []Operation
 
@@ -274,7 +280,13 @@ func reconcileGlobalContainer(
 			continue
 		}
 
-		status := EvalContainerSpecChange(c.Container.ServiceSpec, spec)
+		var status ContainerSpecStatus
+		if forceRecreate {
+			status = ContainerNeedsRecreate
+		} else {
+			status = EvalContainerSpecChange(c.Container.ServiceSpec, spec)
+		}
+
 		if status == ContainerUpToDate {
 			// The container is already running with the same spec.
 			upToDate = true

test/e2e/assert.go

@@ -100,7 +100,7 @@ func assertContainerMatchesSpec(t *testing.T, ctr api.ServiceContainer, spec api
 	assert.Equal(t, portBindings, ctr.HostConfig.PortBindings)
 
 	assert.Equal(t, container.RestartPolicy{
-		Name:              container.RestartPolicyAlways,
+		Name:              container.RestartPolicyUnlessStopped,
 		MaximumRetryCount: 0,
 	}, ctr.HostConfig.RestartPolicy)
 

test/e2e/compose_deploy_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/psviderski/uncloud/internal/ucind"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client/compose"
+	"github.com/psviderski/uncloud/pkg/client/deploy"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -34,14 +35,14 @@ func TestComposeDeployment(t *testing.T) {
 		project, err := compose.LoadProject(ctx, []string{"fixtures/compose-basic.yaml"})
 		require.NoError(t, err)
 
-		deploy, err := compose.NewDeployment(ctx, cli, project)
+		deployment, err := compose.NewDeployment(ctx, cli, project)
 		require.NoError(t, err)
 
-		plan, err := deploy.Plan(ctx)
+		plan, err := deployment.Plan(ctx)
 		require.NoError(t, err)
 		assert.Len(t, plan.Operations, 1, "Expected 1 service to deploy")
 
-		err = deploy.Run(ctx)
+		err = deployment.Run(ctx)
 		require.NoError(t, err)
 
 		svc, err := cli.InspectService(ctx, name)
@@ -71,6 +72,137 @@ func TestComposeDeployment(t *testing.T) {
 		assertServiceMatchesSpec(t, svc, expectedSpec)
 	})
 
+	t.Run("multi-service deployment with redeploy and recreate", func(t *testing.T) {
+		t.Parallel()
+
+		serviceNames := []string{
+			"test-compose-multi-web",
+			"test-compose-multi-api",
+			"test-compose-multi-worker",
+		}
+		t.Cleanup(func() {
+			removeServices(t, cli, serviceNames...)
+		})
+
+		// Initial deployment.
+		project, err := compose.LoadProject(ctx, []string{"fixtures/compose-multi-service.yaml"})
+		require.NoError(t, err)
+
+		deployment, err := compose.NewDeployment(ctx, cli, project)
+		require.NoError(t, err)
+
+		plan, err := deployment.Plan(ctx)
+		require.NoError(t, err)
+		assert.Len(t, plan.Operations, 3, "Expected 3 services to deploy")
+
+		err = deployment.Run(ctx)
+		require.NoError(t, err)
+
+		// Verify web service.
+		webSvc, err := cli.InspectService(ctx, "test-compose-multi-web")
+		require.NoError(t, err)
+		expectedWebSpec := api.ServiceSpec{
+			Name: "test-compose-multi-web",
+			Mode: api.ServiceModeReplicated,
+			Container: api.ContainerSpec{
+				Env: map[string]string{
+					"SERVICE": "web",
+					"VERSION": "1.0",
+				},
+				Image: "portainer/pause:3.9",
+			},
+			Ports: []api.PortSpec{
+				{
+					Hostname:      "multi.example.com",
+					ContainerPort: 80,
+					Protocol:      api.ProtocolHTTPS,
+					Mode:          api.PortModeIngress,
+				},
+			},
+			Replicas: 2,
+		}
+		assertServiceMatchesSpec(t, webSvc, expectedWebSpec)
+
+		// Verify api service.
+		apiSvc, err := cli.InspectService(ctx, "test-compose-multi-api")
+		require.NoError(t, err)
+		expectedApiSpec := api.ServiceSpec{
+			Name: "test-compose-multi-api",
+			Mode: api.ServiceModeReplicated,
+			Container: api.ContainerSpec{
+				Env: map[string]string{
+					"SERVICE": "api",
+					"PORT":    "8080",
+				},
+				Image: "portainer/pause:3.9",
+			},
+			Replicas: 3,
+		}
+		assertServiceMatchesSpec(t, apiSvc, expectedApiSpec)
+
+		// Verify worker service.
+		workerSvc, err := cli.InspectService(ctx, "test-compose-multi-worker")
+		require.NoError(t, err)
+		expectedWorkerSpec := api.ServiceSpec{
+			Name: "test-compose-multi-worker",
+			Mode: api.ServiceModeReplicated,
+			Container: api.ContainerSpec{
+				Env: map[string]string{
+					"SERVICE":     "worker",
+					"CONCURRENCY": "5",
+				},
+				Image: "portainer/pause:3.9",
+			},
+			Replicas: 1,
+		}
+		assertServiceMatchesSpec(t, workerSvc, expectedWorkerSpec)
+
+		// Save container IDs for later verification.
+		containers := serviceContainerIDs(webSvc).
+			Union(serviceContainerIDs(apiSvc)).
+			Union(serviceContainerIDs(workerSvc))
+
+		// Redeploy without changes - should be up to date.
+		redeploy, err := compose.NewDeployment(ctx, cli, project)
+		require.NoError(t, err)
+
+		redeployPlan, err := redeploy.Plan(ctx)
+		require.NoError(t, err)
+		assert.Len(t, redeployPlan.Operations, 0, "Expected no operations - deployment should be up to date")
+
+		// Deploy with ForceRecreate - should recreate all service containers.
+		strategy := &deploy.RollingStrategy{ForceRecreate: true}
+		recreateDeploy, err := compose.NewDeploymentWithStrategy(ctx, cli, project, strategy)
+		require.NoError(t, err)
+
+		recreatePlan, err := recreateDeploy.Plan(ctx)
+		require.NoError(t, err)
+		assert.Len(t, recreatePlan.Operations, 3, "Expected 3 services to be recreated")
+
+		err = recreateDeploy.Run(ctx)
+		require.NoError(t, err)
+
+		// Verify services match the expected specs after recreate.
+		webSvcAfter, err := cli.InspectService(ctx, "test-compose-multi-web")
+		require.NoError(t, err)
+		assertServiceMatchesSpec(t, webSvcAfter, expectedWebSpec)
+
+		apiSvcAfter, err := cli.InspectService(ctx, "test-compose-multi-api")
+		require.NoError(t, err)
+		assertServiceMatchesSpec(t, apiSvcAfter, expectedApiSpec)
+
+		workerSvcAfter, err := cli.InspectService(ctx, "test-compose-multi-worker")
+		require.NoError(t, err)
+		assertServiceMatchesSpec(t, workerSvcAfter, expectedWorkerSpec)
+
+		// Verify that all containers have been recreated.
+		afterContainers := serviceContainerIDs(webSvcAfter).
+			Union(serviceContainerIDs(apiSvcAfter)).
+			Union(serviceContainerIDs(workerSvcAfter))
+		assert.NotEqual(t, containers.ToSlice(), afterContainers.ToSlice(),
+			"Expected containers to be recreated after deployment with ForceRecreate strategy")
+	})
+
 	t.Run("multiple services with volumes", func(t *testing.T) {
 		t.Parallel()
 
@@ -94,10 +226,10 @@ func TestComposeDeployment(t *testing.T) {
 		project, err := compose.LoadProject(ctx, []string{"fixtures/compose-volumes.yaml"})
 		require.NoError(t, err)
 
-		deploy, err := compose.NewDeployment(ctx, cli, project)
+		deployment, err := compose.NewDeployment(ctx, cli, project)
 		require.NoError(t, err)
 
-		_, err = deploy.Plan(ctx)
+		_, err = deployment.Plan(ctx)
 		require.ErrorContains(t, err, "external volumes not found: 'test-compose-volumes-external'")
 
 		externalVolumeOpts := volume.CreateOptions{Name: "test-compose-volumes-external"}
@@ -105,14 +237,14 @@ func TestComposeDeployment(t *testing.T) {
 		require.NoError(t, err)
 
 		// Recreate the deployment as it caches the cluster state.
-		deploy, err = compose.NewDeployment(ctx, cli, project)
+		deployment, err = compose.NewDeployment(ctx, cli, project)
 		require.NoError(t, err)
 
-		plan, err := deploy.Plan(ctx)
+		plan, err := deployment.Plan(ctx)
 		require.NoError(t, err)
 		assert.Len(t, plan.Operations, 5, "Expected 2 volumes creation and 3 services to deploy")
 
-		err = deploy.Run(ctx)
+		err = deployment.Run(ctx)
 		require.NoError(t, err)
 
 		// Verify data1 and data2 volumes have been created.
@@ -247,10 +379,10 @@ func TestComposeDeployment(t *testing.T) {
 			"service3 should be on the same machine as external volume")
 
 		// Verify deployment is up-to-date.
-		deploy, err = compose.NewDeployment(ctx, cli, project)
+		deployment, err = compose.NewDeployment(ctx, cli, project)
 		require.NoError(t, err)
 
-		plan, err = deploy.Plan(ctx)
+		plan, err = deployment.Plan(ctx)
 		require.NoError(t, err)
 		assert.Len(t, plan.Operations, 0, "Expected no new operations after deployment")
 	})
@@ -266,14 +398,14 @@ func TestComposeDeployment(t *testing.T) {
 		project, err := compose.LoadProject(ctx, []string{"fixtures/compose-placement.yaml"})
 		require.NoError(t, err)
 
-		deploy, err := compose.NewDeployment(ctx, cli, project)
+		deployment, err := compose.NewDeployment(ctx, cli, project)
 		require.NoError(t, err)
 
-		plan, err := deploy.Plan(ctx)
+		plan, err := deployment.Plan(ctx)
 		require.NoError(t, err)
 		assert.Len(t, plan.Operations, 1, "Expected 1 service to deploy")
 
-		err = deploy.Run(ctx)
+		err = deployment.Run(ctx)
 		require.NoError(t, err)
 
 		svc, err := cli.InspectService(ctx, name)
@@ -319,14 +451,14 @@ func TestComposeDeployment(t *testing.T) {
 		project, err := compose.LoadProject(ctx, []string{"fixtures/compose-placement-nonexistent.yaml"})
 		require.NoError(t, err)
 
-		deploy, err := compose.NewDeployment(ctx, cli, project)
+		deployment, err := compose.NewDeployment(ctx, cli, project)
 		require.NoError(t, err)
 
-		plan, err := deploy.Plan(ctx)
+		plan, err := deployment.Plan(ctx)
 		require.NoError(t, err)
 		assert.Len(t, plan.Operations, 1, "Expected 1 service to deploy")
 
-		err = deploy.Run(ctx)
+		err = deployment.Run(ctx)
 		require.NoError(t, err)
 
 		svc, err := cli.InspectService(ctx, name)
@@ -367,14 +499,14 @@ func TestComposeDeployment(t *testing.T) {
 		project, err := compose.LoadProject(ctx, []string{"fixtures/compose-placement-comma.yaml"})
 		require.NoError(t, err)
 
-		deploy, err := compose.NewDeployment(ctx, cli, project)
+		deployment, err := compose.NewDeployment(ctx, cli, project)
 		require.NoError(t, err)
 
-		plan, err := deploy.Plan(ctx)
+		plan, err := deployment.Plan(ctx)
 		require.NoError(t, err)
 		assert.Len(t, plan.Operations, 1, "Expected 1 service to deploy")
 
-		err = deploy.Run(ctx)
+		err = deployment.Run(ctx)
 		require.NoError(t, err)
 
 		svc, err := cli.InspectService(ctx, name)

test/e2e/fixtures/compose-multi-service.yaml

@@ -0,0 +1,24 @@
+services:
+  test-compose-multi-web:
+    image: portainer/pause:3.9
+    environment:
+      SERVICE: web
+      VERSION: "1.0"
+    deploy:
+      replicas: 2
+    x-ports:
+      - multi.example.com:80/https
+
+  test-compose-multi-api:
+    image: portainer/pause:3.9
+    environment:
+      SERVICE: api
+      PORT: "8080"
+    deploy:
+      replicas: 3
+
+  test-compose-multi-worker:
+    image: portainer/pause:3.9
+    environment:
+      SERVICE: worker
+      CONCURRENCY: "5"

test/e2e/service_test.go

@@ -279,35 +279,11 @@ func TestDeployment(t *testing.T) {
 
 		svc, err := cli.InspectService(ctx, client.CaddyServiceName)
 		require.NoError(t, err)
-		assert.Equal(t, client.CaddyServiceName, svc.Name)
-		assert.Equal(t, api.ServiceModeGlobal, svc.Mode)
 		assert.Len(t, svc.Containers, 3)
+		assertServiceMatchesSpec(t, svc, deployment.Spec)
 
 		ctr := svc.Containers[0].Container
 		assert.Regexp(t, `^caddy:2\.\d+\.\d+$`, ctr.Config.Image)
-
-		ports, err := ctr.ServicePorts()
-		require.NoError(t, err)
-		expectedPorts := []api.PortSpec{
-			{
-				PublishedPort: 80,
-				ContainerPort: 80,
-				Protocol:      api.ProtocolTCP,
-				Mode:          api.PortModeHost,
-			},
-			{
-				PublishedPort: 443,
-				ContainerPort: 443,
-				Protocol:      api.ProtocolTCP,
-				Mode:          api.PortModeHost,
-			},
-		}
-		assert.Equal(t, expectedPorts, ports)
-
-		assert.Equal(t, container.RestartPolicy{
-			Name:              container.RestartPolicyAlways,
-			MaximumRetryCount: 0,
-		}, ctr.HostConfig.RestartPolicy)
 	})
 
 	t.Run("caddy with machine placement", func(t *testing.T) {