philipp/uncloud
1
0
Fork 0
mirror of https://github.com/psviderski/uncloud.git synced 2026-08-26 11:03:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement container health monitoring and rollback during rolling deployment (closes #24)

Browse Source
This commit is contained in:
Pasha Sviderski
2026-02-27 21:36:40 +10:00
parent 17c1bb5c21
commit 7a9eac593f
7 changed files with 396 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/service.go
+7 -2
View File
@@ -438,9 +438,14 @@ type LogDriver struct {
// UpdateConfig configures how a service is updated during a deployment.
type UpdateConfig struct {
// Order specifies the order of operations during an update.
// Valid values are "start-first" (default for stateless services) and "stop-first" (default for services with volumes).
// Empty value means the strategy will determine the order based on service characteristics.
// Valid values are "start-first" (default for stateless services) and "stop-first" (default for services with
// volumes). Empty value means the strategy will determine the order based on service characteristics.
Order string `json:",omitempty"`
// MonitorPeriod is how long to wait after starting a container before checking that it's still running
// and not restarting. Containers with a health check that become healthy before the period ends succeed early.
// nil means use the default api.DefaultHealthMonitorPeriod.
// Zero skips the monitoring and checks the container's health immediately after starting.
MonitorPeriod *time.Duration `json:",omitempty"`
}
type RunServiceResponse struct {
pkg/client/compose/project.go
+21
View File
@@ -6,13 +6,23 @@ import (
"os"
"path/filepath"
"strings"
"sync"
composecli "github.com/compose-spec/compose-go/v2/cli"
"github.com/compose-spec/compose-go/v2/transform"
"github.com/compose-spec/compose-go/v2/tree"
"github.com/compose-spec/compose-go/v2/types"
"github.com/psviderski/uncloud/pkg/api"
)
var registerComposeOverrides sync.Once
// LoadProject loads a Compose project from the default locations or the given paths.
func LoadProject(ctx context.Context, paths []string, opts ...composecli.ProjectOptionsFn) (*types.Project, error) {
registerComposeOverrides.Do(func() {
transform.RegisterDefaultValue("services.*.deploy.update_config", setUpdateConfigDefaults)
})
defaultOpts := []composecli.ProjectOptionsFn{
// First apply os.Environment, always wins.
composecli.WithOsEnv,
@@ -92,3 +102,14 @@ func removeProjectPrefixFromNames(project *types.Project) {
project.Volumes[name] = vol
}
}
// setUpdateConfigDefaults sets default values for deploy.update_config attributes when not specified in the compose file.
func setUpdateConfigDefaults(data any, _ tree.Path, _ bool) (any, error) {
switch v := data.(type) {
case map[string]any:
if _, ok := v["monitor"]; !ok {
v["monitor"] = api.DefaultHealthMonitorPeriod.String()
}
}
return data, nil
}
pkg/client/compose/service.go
+4 -2
View File
@@ -101,7 +101,6 @@ func ServiceSpecFromCompose(project *types.Project, serviceName string) (api.Ser
return spec, fmt.Errorf("unsupported deploy mode: '%s'", service.Deploy.Mode)
}
// Parse update_config.order
if cfg := service.Deploy.UpdateConfig; cfg != nil {
switch cfg.Order {
case "":
@@ -111,8 +110,11 @@ func ServiceSpecFromCompose(project *types.Project, serviceName string) (api.Ser
case "stop-first":
spec.UpdateConfig.Order = api.UpdateOrderStopFirst
default:
return spec, fmt.Errorf("unsupported update_config.order: '%s'", cfg.Order)
return spec, fmt.Errorf("unsupported deploy.update_config.order: '%s'", cfg.Order)
}
d := time.Duration(cfg.Monitor)
spec.UpdateConfig.MonitorPeriod = &d
}
}
pkg/client/compose/service_test.go
+93 -16
View File
@@ -207,7 +207,8 @@ func TestServiceSpecFromCompose(t *testing.T) {
},
Replicas: 3,
UpdateConfig: api.UpdateConfig{
Order: api.UpdateOrderStopFirst,
Order: api.UpdateOrderStopFirst,
MonitorPeriod: &api.DefaultHealthMonitorPeriod,
},
Volumes: []api.VolumeSpec{
{
@@ -312,6 +313,8 @@ func TestServiceSpecFromCompose(t *testing.T) {
}
func TestServiceSpecFromCompose_Caddy(t *testing.T) {
t.Parallel()
tests := []struct {
name string
composeYAML string
@@ -460,6 +463,8 @@ services:
}
func TestServiceSpecFromCompose_GPUs(t *testing.T) {
t.Parallel()
tests := []struct {
name string
composeYAML string
@@ -647,6 +652,8 @@ services:
}
func TestServiceSpecFromCompose_VolumeDriverOpts(t *testing.T) {
t.Parallel()
tests := []struct {
name string
composeYAML string
@@ -755,6 +762,8 @@ volumes:
}
func TestServiceSpecFromCompose_Ulimits(t *testing.T) {
t.Parallel()
tests := []struct {
name string
composeYAML string
@@ -843,6 +852,8 @@ services:
}
func TestServiceSpecFromCompose_UpdateConfig(t *testing.T) {
t.Parallel()
tests := []struct {
name string
composeYAML string
@@ -858,6 +869,33 @@ services:
`,
expected: api.UpdateConfig{},
},
{
name: "empty update_config",
composeYAML: `
services:
test:
image: nginx
deploy:
update_config: {}
`,
expected: api.UpdateConfig{
MonitorPeriod: &api.DefaultHealthMonitorPeriod,
},
},
{
name: "update_config with unsupported attributes ignored",
composeYAML: `
services:
test:
image: nginx
deploy:
update_config:
parallelism: 1
`,
expected: api.UpdateConfig{
MonitorPeriod: &api.DefaultHealthMonitorPeriod,
},
},
{
name: "update_config with stop-first order",
composeYAML: `
@@ -869,7 +907,8 @@ services:
order: stop-first
`,
expected: api.UpdateConfig{
Order: api.UpdateOrderStopFirst,
Order: api.UpdateOrderStopFirst,
MonitorPeriod: &api.DefaultHealthMonitorPeriod,
},
},
{
@@ -883,7 +922,8 @@ services:
order: start-first
`,
expected: api.UpdateConfig{
Order: api.UpdateOrderStartFirst,
Order: api.UpdateOrderStartFirst,
MonitorPeriod: &api.DefaultHealthMonitorPeriod,
},
},
{
@@ -898,18 +938,6 @@ services:
`,
expectError: true,
},
{
name: "update_config with empty order",
composeYAML: `
services:
test:
image: nginx
deploy:
update_config:
parallelism: 1
`,
expected: api.UpdateConfig{},
},
{
name: "update_config with replicas and order",
composeYAML: `
@@ -922,7 +950,52 @@ services:
order: stop-first
`,
expected: api.UpdateConfig{
Order: api.UpdateOrderStopFirst,
Order: api.UpdateOrderStopFirst,
MonitorPeriod: &api.DefaultHealthMonitorPeriod,
},
},
{
name: "update_config with custom monitor",
composeYAML: `
services:
test:
image: nginx
deploy:
update_config:
monitor: 10s
`,
expected: api.UpdateConfig{
MonitorPeriod: api.AsPtr(10 * time.Second),
},
},
{
name: "update_config with monitor and order",
composeYAML: `
services:
test:
image: nginx
deploy:
update_config:
order: start-first
monitor: 30s
`,
expected: api.UpdateConfig{
Order: api.UpdateOrderStartFirst,
MonitorPeriod: api.AsPtr(30 * time.Second),
},
},
{
name: "update_config with zero monitor skips monitoring",
composeYAML: `
services:
test:
image: nginx
deploy:
update_config:
monitor: 0s
`,
expected: api.UpdateConfig{
MonitorPeriod: api.AsPtr(time.Duration(0)),
},
},
}
@@ -949,6 +1022,8 @@ services:
}
func TestServiceSpecFromCompose_XMachinesPlacement(t *testing.T) {
t.Parallel()
tests := []struct {
name string
composeYAML string
@@ -1107,6 +1182,8 @@ services:
}
func TestServiceSpecFromCompose_Devices(t *testing.T) {
t.Parallel()
tests := []struct {
name string
composeYAML string
pkg/client/deploy/operation/container.go
+42 -5
View File
@@ -4,7 +4,9 @@ import (
"context"
"fmt"
"github.com/docker/compose/v2/pkg/progress"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/pkg/stringid"
"github.com/psviderski/uncloud/pkg/api"
)
@@ -24,7 +26,11 @@ func (o *RunContainerOperation) Execute(ctx context.Context, cli Client) error {
return fmt.Errorf("start container: %w", err)
}
// TODO: wait for the container to become healthy
opts := api.WaitContainerHealthyOptions{MonitorPeriod: o.Spec.UpdateConfig.MonitorPeriod}
if err = cli.WaitContainerHealthy(ctx, o.ServiceID, resp.ID, opts); err != nil {
return fmt.Errorf("container '%s/%s' failed to become healthy: %w",
o.Spec.Name, stringid.TruncateID(resp.ID), err)
}
return nil
}
@@ -111,14 +117,12 @@ func (o *ReplaceContainerOperation) Execute(ctx context.Context, cli Client) err
stopFirst := o.Order == api.UpdateOrderStopFirst
if stopFirst {
// TODO: inspect and remember the current status of the old container.
if err := cli.StopContainer(ctx, o.ServiceID, o.OldContainer.ID, container.StopOptions{}); err != nil {
return fmt.Errorf("stop old container: %w", err)
}
}
// TODO: Rollback support - if new container fails to start, stop new, collect logs, and restart old container (#24)
// TODO: When parallelism is added, rollback becomes more complex - need to track which containers
// were stopped and restore them all on failure
resp, err := cli.CreateContainer(ctx, o.ServiceID, o.Spec, o.MachineID)
if err != nil {
return fmt.Errorf("create container: %w", err)
@@ -127,11 +131,44 @@ func (o *ReplaceContainerOperation) Execute(ctx context.Context, cli Client) err
return fmt.Errorf("start container: %w", err)
}
// TODO: wait for the container to become healthy. If unhealthy, stop new container, collect logs, and start old.
opts := api.WaitContainerHealthyOptions{MonitorPeriod: o.Spec.UpdateConfig.MonitorPeriod}
if err = cli.WaitContainerHealthy(ctx, o.ServiceID, resp.ID, opts); err != nil {
// New container failed to become healthy. Stop it and roll back to the previous container.
// Don't remove the new stopped container to allow users to inspect logs and state.
// TODO: collect logs from the new container and include in the error message to speed up debugging.
// Use context without progress to not overwrite the container Unhealthy status with Stopped.
ctxWithoutProgress := progress.WithContextWriter(ctx, nil)
_ = cli.StopContainer(ctxWithoutProgress, o.ServiceID, resp.ID, container.StopOptions{})
newCtr := fmt.Sprintf("%s/%s", o.Spec.Name, stringid.TruncateID(resp.ID))
oldCtr := fmt.Sprintf("%s/%s", o.OldContainer.ServiceSpec.Name, o.OldContainer.ShortID())
if stopFirst {
// Restart the old container since we stopped it earlier.
// TODO: restart only if the old container was running before we stopped it to avoid starting the failed
// or intentionally stopped container.
if rollbackErr := cli.StartContainer(ctx, o.ServiceID, o.OldContainer.ID); rollbackErr != nil {
return fmt.Errorf(
"new container '%s' failed to become healthy: %w; "+
"rolled back to previous container '%s' but failed to restart it: %w",
newCtr, rollbackErr, oldCtr, err,
)
}
}
return fmt.Errorf("new container '%s' failed to become healthy: %w. Rolled back to previous container '%s'. "+
"New container has been stopped and is available for inspection. Fetch logs with 'uc logs %s'",
newCtr, err, oldCtr, o.Spec.Name)
}
// For start-first, we need to stop before removing.
// For stop-first, the container is already stopped.
if !stopFirst {
// TODO: the new container is propagated to Caddy upstreams through the cluster store asynchronously.
// There still might be a brief downtime (for a 1 replica service) when Caddy doesn't know about
// the new container but we're stopping the old container. We should somehow ensure Caddy is updated
// with the new container before we stop the old one to avoid this downtime.
if err := cli.StopContainer(ctx, o.ServiceID, o.OldContainer.ID, container.StopOptions{}); err != nil {
return fmt.Errorf("stop old container: %w", err)
}