feat: add --skip-health flag to bypass health monitoring during container deployment

This commit is contained in:
Pasha Sviderski
2026-02-28 16:26:48 +10:00
parent e6705599b3
commit ce3cb8a34f
4 changed files with 80 additions and 54 deletions
+14 -9
View File
@@ -21,12 +21,13 @@ import (
type deployOptions struct { type deployOptions struct {
cli.BuildServicesOptions cli.BuildServicesOptions
files []string files []string
profiles []string profiles []string
services []string services []string
noBuild bool noBuild bool
recreate bool recreate bool
yes bool skipHealth bool
yes bool
} }
// NewDeployCommand creates a new command to deploy services from a Compose file. // NewDeployCommand creates a new command to deploy services from a Compose file.
@@ -61,6 +62,10 @@ func NewDeployCommand() *cobra.Command {
"One or more Compose profiles to enable.") "One or more Compose profiles to enable.")
cmd.Flags().BoolVar(&opts.recreate, "recreate", false, cmd.Flags().BoolVar(&opts.recreate, "recreate", false,
"Recreate containers even if their configuration and image haven't changed.") "Recreate containers even if their configuration and image haven't changed.")
cmd.Flags().BoolVar(&opts.skipHealth, "skip-health", false,
"Skip the monitoring period and health checks after starting new containers. Useful for faster emergency "+
"deployments.\n"+
"Warning: This may cause downtime if new containers fail to start properly.")
cmd.Flags().BoolVarP(&opts.yes, "yes", "y", false, cmd.Flags().BoolVarP(&opts.yes, "yes", "y", false,
"Auto-confirm deployment plan. Should be explicitly set when running non-interactively,\n"+ "Auto-confirm deployment plan. Should be explicitly set when running non-interactively,\n"+
"e.g., in CI/CD pipelines. [$UNCLOUD_AUTO_CONFIRM]") "e.g., in CI/CD pipelines. [$UNCLOUD_AUTO_CONFIRM]")
@@ -149,9 +154,9 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
fmt.Println() fmt.Println()
} }
var strategy deploy.Strategy strategy := &deploy.RollingStrategy{
if opts.recreate { ForceRecreate: opts.recreate,
strategy = &deploy.RollingStrategy{ForceRecreate: true} SkipHealthMonitor: opts.skipHealth,
} }
composeDeploy, err := compose.NewDeploymentWithStrategy(ctx, clusterClient, project, strategy) composeDeploy, err := compose.NewDeploymentWithStrategy(ctx, clusterClient, project, strategy)
if err != nil { if err != nil {
+33 -23
View File
@@ -15,6 +15,8 @@ type RunContainerOperation struct {
ServiceID string ServiceID string
Spec api.ServiceSpec Spec api.ServiceSpec
MachineID string MachineID string
// SkipHealthMonitor skips the monitoring period and health checks after starting a container.
SkipHealthMonitor bool
} }
func (o *RunContainerOperation) Execute(ctx context.Context, cli Client) error { func (o *RunContainerOperation) Execute(ctx context.Context, cli Client) error {
@@ -26,6 +28,10 @@ func (o *RunContainerOperation) Execute(ctx context.Context, cli Client) error {
return fmt.Errorf("start container: %w", err) return fmt.Errorf("start container: %w", err)
} }
if o.SkipHealthMonitor {
return nil
}
opts := api.WaitContainerHealthyOptions{MonitorPeriod: o.Spec.UpdateConfig.MonitorPeriod} opts := api.WaitContainerHealthyOptions{MonitorPeriod: o.Spec.UpdateConfig.MonitorPeriod}
if err = cli.WaitContainerHealthy(ctx, o.ServiceID, resp.ID, opts); err != nil { if err = cli.WaitContainerHealthy(ctx, o.ServiceID, resp.ID, opts); err != nil {
return fmt.Errorf("container '%s/%s' failed to become healthy: %w", return fmt.Errorf("container '%s/%s' failed to become healthy: %w",
@@ -111,6 +117,8 @@ type ReplaceContainerOperation struct {
OldContainer api.ServiceContainer OldContainer api.ServiceContainer
// Order specifies the update order: "start-first" or "stop-first". // Order specifies the update order: "start-first" or "stop-first".
Order string Order string
// SkipHealthMonitor skips the monitoring period and health checks after starting a new container.
SkipHealthMonitor bool
} }
func (o *ReplaceContainerOperation) Execute(ctx context.Context, cli Client) error { func (o *ReplaceContainerOperation) Execute(ctx context.Context, cli Client) error {
@@ -139,34 +147,36 @@ func (o *ReplaceContainerOperation) Execute(ctx context.Context, cli Client) err
return fmt.Errorf("start new container: %w", err) return fmt.Errorf("start new container: %w", err)
} }
opts := api.WaitContainerHealthyOptions{MonitorPeriod: o.Spec.UpdateConfig.MonitorPeriod} if !o.SkipHealthMonitor {
if err = cli.WaitContainerHealthy(ctx, o.ServiceID, resp.ID, opts); err != nil { opts := api.WaitContainerHealthyOptions{MonitorPeriod: o.Spec.UpdateConfig.MonitorPeriod}
// New container failed to become healthy. Stop it and roll back to the previous container. if err = cli.WaitContainerHealthy(ctx, o.ServiceID, resp.ID, opts); err != nil {
// Don't remove the new stopped container to allow users to inspect logs and state. // New container failed to become healthy. Stop it and roll back to the previous container.
// TODO: collect logs from the new container and include in the error message to speed up debugging. // Don't remove the new stopped container to allow users to inspect logs and state.
// TODO: collect logs from the new container and include in the error message to speed up debugging.
// Use context without progress to not overwrite the container Unhealthy status with Stopped. // Use context without progress to not overwrite the container Unhealthy status with Stopped.
ctxWithoutProgress := progress.WithContextWriter(ctx, nil) ctxWithoutProgress := progress.WithContextWriter(ctx, nil)
_ = cli.StopContainer(ctxWithoutProgress, o.ServiceID, resp.ID, container.StopOptions{}) _ = cli.StopContainer(ctxWithoutProgress, o.ServiceID, resp.ID, container.StopOptions{})
newCtr := fmt.Sprintf("%s/%s", o.Spec.Name, stringid.TruncateID(resp.ID)) newCtr := fmt.Sprintf("%s/%s", o.Spec.Name, stringid.TruncateID(resp.ID))
healthErr := fmt.Errorf( healthErr := fmt.Errorf(
"new container '%s' failed to become healthy: %w. "+ "new container '%s' failed to become healthy: %w. "+
"It's stopped and available for inspection. Fetch logs with 'uc logs %s'", "It's stopped and available for inspection. Fetch logs with 'uc logs %s'",
newCtr, err, o.Spec.Name, newCtr, err, o.Spec.Name,
) )
if stopFirst && wasRunning { if stopFirst && wasRunning {
// Restart the old container only if it was running before we stopped it. // Restart the old container only if it was running before we stopped it.
oldCtr := fmt.Sprintf("%s/%s", o.OldContainer.ServiceSpec.Name, o.OldContainer.ShortID()) oldCtr := fmt.Sprintf("%s/%s", o.OldContainer.ServiceSpec.Name, o.OldContainer.ShortID())
if rollbackErr := cli.StartContainer(ctx, o.ServiceID, o.OldContainer.ID); rollbackErr != nil { if rollbackErr := cli.StartContainer(ctx, o.ServiceID, o.OldContainer.ID); rollbackErr != nil {
return fmt.Errorf("%w. Rolled back to old container '%s' but failed to restart it: %w", return fmt.Errorf("%w. Rolled back to old container '%s' but failed to restart it: %w",
healthErr, oldCtr, rollbackErr) healthErr, oldCtr, rollbackErr)
}
return fmt.Errorf("%w. Rolled back to old container '%s'", healthErr, oldCtr)
} }
return fmt.Errorf("%w. Rolled back to old container '%s'", healthErr, oldCtr)
}
return healthErr return healthErr
}
} }
// For start-first, we need to stop before removing. // For start-first, we need to stop before removing.
+30 -21
View File
@@ -29,6 +29,8 @@ type RollingStrategy struct {
// ForceRecreate indicates whether all containers should be recreated during the deployment, // ForceRecreate indicates whether all containers should be recreated during the deployment,
// regardless of whether their specifications have changed. // regardless of whether their specifications have changed.
ForceRecreate bool ForceRecreate bool
// SkipHealthMonitor skips the monitoring period and health checks for faster emergency deployments.
SkipHealthMonitor bool
// state is the current and planned state of the cluster used for scheduling decisions. // state is the current and planned state of the cluster used for scheduling decisions.
state *scheduler.ClusterState state *scheduler.ClusterState
@@ -146,9 +148,10 @@ func (s *RollingStrategy) planReplicated(svc *api.Service, spec api.ServiceSpec)
if len(containers) == 0 { if len(containers) == 0 {
// No more existing containers on this machine, create a new one. // No more existing containers on this machine, create a new one.
plan.Operations = append(plan.Operations, &operation.RunContainerOperation{ plan.Operations = append(plan.Operations, &operation.RunContainerOperation{
ServiceID: plan.ServiceID, ServiceID: plan.ServiceID,
Spec: spec, Spec: spec,
MachineID: m.Id, MachineID: m.Id,
SkipHealthMonitor: s.SkipHealthMonitor,
}) })
continue continue
} }
@@ -166,11 +169,12 @@ func (s *RollingStrategy) planReplicated(svc *api.Service, spec api.ServiceSpec)
// Replace the old container with a new one. // Replace the old container with a new one.
order := determineUpdateOrder(ctr, spec) order := determineUpdateOrder(ctr, spec)
plan.Operations = append(plan.Operations, &operation.ReplaceContainerOperation{ plan.Operations = append(plan.Operations, &operation.ReplaceContainerOperation{
ServiceID: plan.ServiceID, ServiceID: plan.ServiceID,
Spec: spec, Spec: spec,
MachineID: m.Id, MachineID: m.Id,
OldContainer: ctr, OldContainer: ctr,
Order: order, Order: order,
SkipHealthMonitor: s.SkipHealthMonitor,
}) })
} }
@@ -216,7 +220,8 @@ func (s *RollingStrategy) planGlobal(svc *api.Service, spec api.ServiceSpec) (Pl
for _, m := range availableMachines { for _, m := range availableMachines {
containers := containersOnMachine[m.Info.Id] containers := containersOnMachine[m.Info.Id]
ops, err := reconcileGlobalContainer(containers, spec, plan.ServiceID, m.Info.Id, s.ForceRecreate) ops, err := reconcileGlobalContainer(
containers, spec, plan.ServiceID, m.Info.Id, s.ForceRecreate, s.SkipHealthMonitor)
if err != nil { if err != nil {
return plan, err return plan, err
} }
@@ -242,16 +247,18 @@ func (s *RollingStrategy) planGlobal(svc *api.Service, spec api.ServiceSpec) (Pl
// It ensures exactly one container with the desired spec is running on the machine by creating a new container and // It ensures exactly one container with the desired spec is running on the machine by creating a new container and
// removing old ones. If there is a host port conflict, it stops the old container before starting a new one. // removing old ones. If there is a host port conflict, it stops the old container before starting a new one.
func reconcileGlobalContainer( func reconcileGlobalContainer(
containers []api.MachineServiceContainer, spec api.ServiceSpec, serviceID, machineID string, forceRecreate bool, containers []api.MachineServiceContainer, spec api.ServiceSpec, serviceID, machineID string,
forceRecreate, skipHealthCheck bool,
) ([]operation.Operation, error) { ) ([]operation.Operation, error) {
var ops []operation.Operation var ops []operation.Operation
if len(containers) == 0 { if len(containers) == 0 {
// No containers on this machine, create a new one. // No containers on this machine, create a new one.
ops = append(ops, &operation.RunContainerOperation{ ops = append(ops, &operation.RunContainerOperation{
ServiceID: serviceID, ServiceID: serviceID,
Spec: spec, Spec: spec,
MachineID: machineID, MachineID: machineID,
SkipHealthMonitor: skipHealthCheck,
}) })
return ops, nil return ops, nil
} }
@@ -322,11 +329,12 @@ func reconcileGlobalContainer(
// Replace the running container with a new one. // Replace the running container with a new one.
order := determineUpdateOrder(containerToReplace.Container, spec) order := determineUpdateOrder(containerToReplace.Container, spec)
ops = append(ops, &operation.ReplaceContainerOperation{ ops = append(ops, &operation.ReplaceContainerOperation{
ServiceID: serviceID, ServiceID: serviceID,
Spec: spec, Spec: spec,
MachineID: machineID, MachineID: machineID,
OldContainer: containerToReplace.Container, OldContainer: containerToReplace.Container,
Order: order, Order: order,
SkipHealthMonitor: skipHealthCheck,
}) })
// Remove any other containers (there shouldn't be any in normal operation). // Remove any other containers (there shouldn't be any in normal operation).
@@ -342,9 +350,10 @@ func reconcileGlobalContainer(
} else { } else {
// No running containers, create a new one and remove all stopped containers. // No running containers, create a new one and remove all stopped containers.
ops = append(ops, &operation.RunContainerOperation{ ops = append(ops, &operation.RunContainerOperation{
ServiceID: serviceID, ServiceID: serviceID,
Spec: spec, Spec: spec,
MachineID: machineID, MachineID: machineID,
SkipHealthMonitor: skipHealthCheck,
}) })
for _, c := range containers { for _, c := range containers {
ops = append(ops, &operation.RemoveContainerOperation{ ops = append(ops, &operation.RemoveContainerOperation{
+3 -1
View File
@@ -401,7 +401,9 @@ func TestReconcileGlobalContainer(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
ops, err := reconcileGlobalContainer(tt.containers, tt.spec, "service-1", "machine-1", tt.forceRecreate) ops, err := reconcileGlobalContainer(
tt.containers, tt.spec, "service-1", "machine-1", tt.forceRecreate, false,
)
assert.NoError(t, err) assert.NoError(t, err)
assertOperationsEqual(t, tt.expectedOps, ops) assertOperationsEqual(t, tt.expectedOps, ops)
}) })