feat(labels): support service and container labels, update labels and cpu/memory in-place

This commit is contained in:
Pasha Sviderski
2026-11-02 14:57:33 +10:00
parent b214d23f21
commit 81f30aac26
30 changed files with 1727 additions and 286 deletions
+1
View File
@@ -30,6 +30,7 @@ type ContainerClient interface {
StartContainer(ctx context.Context, serviceNameOrID, containerNameOrID string) error
StopContainer(ctx context.Context, serviceNameOrID, containerNameOrID string, opts container.StopOptions) error
RemoveContainer(ctx context.Context, serviceNameOrID, containerNameOrID string, opts container.RemoveOptions) error
UpdateContainer(ctx context.Context, serviceNameOrID, containerNameOrID string, opts UpdateContainerOptions) error
WaitContainerHealthy(
ctx context.Context, serviceNameOrID, containerNameOrID string, opts WaitContainerHealthyOptions,
) error
+33
View File
@@ -19,6 +19,11 @@ const (
// in internal/machine/docker/manager.go.
DockerNetworkName = "uncloud"
// LabelReservedPrefix is the prefix of labels reserved for Uncloud that users can't set on services and containers.
LabelReservedPrefix = "uncloud."
// LabelDaemonReservedPrefix is the prefix of labels reserved for the Uncloud daemon system containers.
LabelDaemonReservedPrefix = "uncloudd."
// LabelDaemonManaged marks system containers as managed by the Uncloud daemon.
LabelDaemonManaged = "uncloudd.managed"
@@ -175,6 +180,19 @@ type CreateContainerResponse struct {
Name string
}
// UpdateContainerOptions defines the properties of a service container to update in place without recreating it.
// Nil fields are left unchanged.
type UpdateContainerOptions struct {
// CPU is the new CPU limit in nanocores. The limit can be changed but not removed.
CPU *int64
// Memory is the new memory limit in bytes. The limit can be changed but not removed.
Memory *int64
// MemoryReservation is the new memory reservation in bytes. The reservation can be changed but not removed.
MemoryReservation *int64
// ServiceLabels replaces all service labels. A pointer to an empty map removes all service labels.
ServiceLabels *map[string]string
}
type ServiceContainer struct {
Container
ServiceSpec ServiceSpec
@@ -195,6 +213,21 @@ func (c *ServiceContainer) ServiceName() string {
return c.Config.Labels[LabelServiceName]
}
// AppliedResources returns the resources from the service spec with the CPU and memory limits that Docker currently
// applies to the container. The applied limits may differ from the spec if an in-place update was interrupted or
// the container was updated outside of Uncloud.
func (c *ServiceContainer) AppliedResources() ContainerResources {
resources := c.ServiceSpec.Container.Resources
if c.ContainerJSONBase == nil || c.HostConfig == nil {
return resources
}
resources.CPU = c.HostConfig.NanoCPUs
resources.Memory = c.HostConfig.Memory
resources.MemoryReservation = c.HostConfig.MemoryReservation
return resources
}
// ServiceMode returns the replication mode of the service this container belongs to.
func (c *ServiceContainer) ServiceMode() string {
return c.ServiceSpec.Mode
+33
View File
@@ -1,7 +1,11 @@
package api
import (
"errors"
"github.com/docker/docker/api/types/container"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
)
const (
@@ -27,6 +31,35 @@ type ContainerResources struct {
Ulimits map[string]Ulimit
}
// Equals returns true if the resources are equal, treating nil and empty slices and maps as equal.
func (r ContainerResources) Equals(other ContainerResources) bool {
return cmp.Equal(r, other, cmpopts.EquateEmpty())
}
// ValidateUpdateInPlace returns an error explaining why a container created with these resources can't be updated
// to the new resources without recreating it. It returns nil if the container can be updated in place.
// Docker can only change the CPU and memory limits of an existing container and can't remove a limit once it's set.
func (r ContainerResources) ValidateUpdateInPlace(new ContainerResources) error {
// Docker treats zero values as unchanged when updating a container.
if r.CPU != 0 && new.CPU == 0 {
return errors.New("CPU limit can't be removed")
}
if r.Memory != 0 && new.Memory == 0 {
return errors.New("memory limit can't be removed")
}
if r.MemoryReservation != 0 && new.MemoryReservation == 0 {
return errors.New("memory reservation can't be removed")
}
r.CPU = new.CPU
r.Memory = new.Memory
r.MemoryReservation = new.MemoryReservation
if !r.Equals(new) {
return errors.New("only CPU and memory limits can be changed")
}
return nil
}
// DeviceMapping represents a device mapping between host and container.
type DeviceMapping struct {
// HostPath is the path to the device on the host.
+72
View File
@@ -0,0 +1,72 @@
package api
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestContainerResources_ValidateUpdateInPlace(t *testing.T) {
t.Parallel()
current := ContainerResources{
CPU: Core,
Memory: 512 * 1024 * 1024,
MemoryReservation: 256 * 1024 * 1024,
}
tests := []struct {
name string
new func(r *ContainerResources)
wantErr string
}{
{
name: "same resources",
new: func(*ContainerResources) {},
},
{
name: "change CPU and memory limits",
new: func(r *ContainerResources) {
r.CPU = 2 * Core
r.Memory = 1024 * 1024 * 1024
r.MemoryReservation = 128 * 1024 * 1024
},
},
{
name: "remove CPU limit",
new: func(r *ContainerResources) { r.CPU = 0 },
wantErr: "CPU limit can't be removed",
},
{
name: "remove memory limit",
new: func(r *ContainerResources) { r.Memory = 0 },
wantErr: "memory limit can't be removed",
},
{
name: "remove memory reservation",
new: func(r *ContainerResources) { r.MemoryReservation = 0 },
wantErr: "memory reservation can't be removed",
},
{
name: "change shared memory",
new: func(r *ContainerResources) { r.SharedMemory = 64 * 1024 * 1024 },
wantErr: "only CPU and memory limits can be changed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
newResources := current
tt.new(&newResources)
err := current.ValidateUpdateInPlace(newResources)
if tt.wantErr == "" {
assert.NoError(t, err)
} else {
assert.EqualError(t, err, tt.wantErr)
}
})
}
}
+29
View File
@@ -77,6 +77,9 @@ type ServiceSpec struct {
Configs []ConfigSpec
// Container defines the desired state of each container in the service.
Container ContainerSpec
// Labels are user-defined key-value metadata attached to the service. Unlike container labels, they're not
// applied to Docker containers. Keys must not use the reserved "uncloud." prefix.
Labels map[string]string
// Mode is the replication mode of the service. Default is ServiceModeReplicated if empty.
Mode string
Name string
@@ -157,6 +160,9 @@ func (s *ServiceSpec) Validate() error {
if err := s.Container.Validate(); err != nil {
return err
}
if err := ValidateLabels(s.Labels); err != nil {
return fmt.Errorf("invalid service labels: %w", err)
}
switch s.Mode {
case "", ServiceModeGlobal, ServiceModeReplicated:
@@ -243,6 +249,7 @@ func (s *ServiceSpec) Clone() ServiceSpec {
spec.Caddy = &caddyCopy
}
spec.Container = s.Container.Clone()
spec.Labels = maps.Clone(s.Labels)
spec.PreDeploy = s.PreDeploy.Clone()
if s.Ports != nil {
@@ -279,6 +286,10 @@ type ContainerSpec struct {
Image string
// Run a custom init inside the container. If nil, use the daemon's configured settings.
Init *bool
// Labels are user-defined key-value metadata applied as Docker labels to each container in the service.
// Docker labels are immutable so changing them requires recreating the containers.
// Keys must not use the reserved "uncloud." prefix.
Labels map[string]string
// LogDriver overrides the default logging driver for the container. Each Docker daemon can have its own default.
LogDriver *LogDriver
// PidMode sets the PID namespace mode for the container. Currently only "" or "host" is supported.
@@ -333,6 +344,9 @@ func (s *ContainerSpec) Validate() error {
if _, err := reference.ParseDockerRef(s.Image); err != nil {
return fmt.Errorf("invalid image '%s': %w", s.Image, err)
}
if err := ValidateLabels(s.Labels); err != nil {
return fmt.Errorf("invalid container labels: %w", err)
}
for _, m := range s.VolumeMounts {
if err := m.Validate(); err != nil {
@@ -390,6 +404,7 @@ func (s *ContainerSpec) Clone() ContainerSpec {
hc.Test = slices.Clone(s.Healthcheck.Test)
spec.Healthcheck = &hc
}
spec.Labels = maps.Clone(s.Labels)
if s.LogDriver != nil {
logDriver := *s.LogDriver
if s.LogDriver.Options != nil {
@@ -428,6 +443,20 @@ func (s *ContainerSpec) Clone() ContainerSpec {
return spec
}
// ValidateLabels checks that label keys are non-empty and don't use the reserved Uncloud prefixes.
func ValidateLabels(labels map[string]string) error {
for k := range labels {
if strings.TrimSpace(k) == "" {
return fmt.Errorf("label key must not be empty")
}
if strings.HasPrefix(k, LabelReservedPrefix) || strings.HasPrefix(k, LabelDaemonReservedPrefix) {
return fmt.Errorf("label '%s' cannot use the reserved prefix '%s' or '%s'",
k, LabelReservedPrefix, LabelDaemonReservedPrefix)
}
}
return nil
}
type EnvVars map[string]string
// ToSlice converts the environment variables to a slice of strings in the format "key=value".
+46 -2
View File
@@ -63,6 +63,47 @@ func TestServiceSpec_Validate_Name(t *testing.T) {
}
}
func TestServiceSpec_Validate_Labels(t *testing.T) {
t.Parallel()
tests := []struct {
name string
labels map[string]string
wantErr string
}{
{name: "nil labels"},
{name: "valid labels", labels: map[string]string{"com.example.team": "web", "tier": ""}},
{name: "prefix without dot", labels: map[string]string{"uncloudfoo": "bar"}},
{name: "empty key", labels: map[string]string{"": "bar"}, wantErr: "must not be empty"},
{name: "reserved prefix", labels: map[string]string{LabelManaged: ""}, wantErr: "reserved prefix"},
{name: "reserved daemon prefix", labels: map[string]string{"uncloudd.foo": "bar"}, wantErr: "reserved prefix"},
}
for _, tt := range tests {
t.Run(tt.name+"/service", func(t *testing.T) {
t.Parallel()
spec := ServiceSpec{Container: ContainerSpec{Image: "nginx:latest"}, Labels: tt.labels}
err := spec.Validate()
if tt.wantErr != "" {
require.ErrorContains(t, err, "invalid service labels")
require.ErrorContains(t, err, tt.wantErr)
} else {
require.NoError(t, err)
}
})
t.Run(tt.name+"/container", func(t *testing.T) {
t.Parallel()
spec := ServiceSpec{Container: ContainerSpec{Image: "nginx:latest", Labels: tt.labels}}
err := spec.Validate()
if tt.wantErr != "" {
require.ErrorContains(t, err, "invalid container labels")
require.ErrorContains(t, err, tt.wantErr)
} else {
require.NoError(t, err)
}
})
}
}
func TestServiceSpec_Validate_CaddyAndPorts(t *testing.T) {
tests := []struct {
name string
@@ -267,8 +308,9 @@ func TestContainerSpec_Clone(t *testing.T) {
"FOO": "bar",
"BAZ": "qux",
},
Image: "nginx:latest",
Init: new(true),
Image: "nginx:latest",
Init: new(true),
Labels: map[string]string{"com.example.team": "web"},
LogDriver: &LogDriver{
Name: "json-file",
Options: map[string]string{
@@ -313,6 +355,7 @@ func TestContainerSpec_Clone(t *testing.T) {
original.Command[0] = stringModified
original.Entrypoint[0] = stringModified
original.Env["FOO"] = stringModified
original.Labels["com.example.team"] = stringModified
original.LogDriver.Options["max-size"] = stringModified
original.Volumes[0] = stringModified
original.VolumeMounts[0].ContainerPath = stringModified
@@ -328,6 +371,7 @@ func TestContainerSpec_Clone(t *testing.T) {
assert.Equal(t, "NET_ADMIN", cloned.CapAdd[0])
assert.Equal(t, "ALL", cloned.CapDrop[0])
assert.Equal(t, "sh", cloned.Command[0])
assert.Equal(t, "web", cloned.Labels["com.example.team"])
assert.Equal(t, "/bin/bash", cloned.Entrypoint[0])
assert.Equal(t, "bar", cloned.Env["FOO"])
assert.Equal(t, "qux", cloned.Env["BAZ"])
+8 -1
View File
@@ -55,7 +55,7 @@ func (p *Plan) Format() string {
// formatSummary counts all operations across the plan and renders the summary footer.
func (p *Plan) formatSummary() string {
var createCount, startFirstCount, stopFirstCount, removeCount int
var createCount, startFirstCount, stopFirstCount, updateCount, removeCount int
machines := make(map[string]struct{})
for _, op := range p.Volumes {
@@ -76,6 +76,9 @@ func (p *Plan) formatSummary() string {
} else {
startFirstCount++
}
case *operation.UpdateContainerOperation:
machines[o.MachineID] = struct{}{}
updateCount++
case *operation.RemoveContainerOperation:
machines[o.MachineID] = struct{}{}
removeCount++
@@ -99,6 +102,10 @@ func (p *Plan) formatSummary() string {
parts = append(parts,
tui.BoldYellow.Render(strconv.Itoa(stopFirstCount))+" "+tui.Yellow.Render("replace (stop-first)"))
}
if updateCount > 0 {
parts = append(parts,
tui.BoldYellow.Render(strconv.Itoa(updateCount))+" "+tui.Yellow.Render("update (in place)"))
}
if removeCount > 0 {
parts = append(parts,
tui.BoldRed.Render(strconv.Itoa(removeCount))+" "+tui.Red.Render("remove"))
+30
View File
@@ -0,0 +1,30 @@
package compose
import (
"testing"
"github.com/charmbracelet/x/ansi"
"github.com/psviderski/uncloud/pkg/client/deploy"
"github.com/psviderski/uncloud/pkg/client/operation"
"github.com/stretchr/testify/assert"
)
func TestPlan_FormatSummary(t *testing.T) {
t.Parallel()
web := &deploy.ServicePlan{}
web.Operations = []operation.Operation{
&operation.UpdateContainerOperation{MachineID: "m-1"},
&operation.RemoveContainerOperation{MachineID: "m-2"},
}
db := &deploy.ServicePlan{}
db.Operations = []operation.Operation{
&operation.UpdateContainerOperation{MachineID: "m-2"},
}
plan := &Plan{
Volumes: []*operation.CreateVolumeOperation{{MachineID: "m-3"}},
Services: []*deploy.ServicePlan{web, db},
}
assert.Equal(t, "1 create · 2 update (in place) · 1 remove · across 3 machines", ansi.Strip(plan.formatSummary()))
}
-14
View File
@@ -344,20 +344,6 @@ configs:
warnCount: 1,
warnContains: []string{"container_name"},
},
{
name: "deploy labels",
composeYAML: `services:
app:
image: myapp:latest
deploy:
mode: replicated
labels:
- foo
replicas: 2
`,
warnCount: 1,
warnContains: []string{"deploy labels"},
},
{
name: "deploy restart_policy",
composeYAML: `services:
+8 -6
View File
@@ -66,6 +66,10 @@ func ServiceSpecFromCompose(project *types.Project, serviceName string) (api.Ser
Mode: api.ServiceModeReplicated,
}
if labels := mergeLabels(service.Labels, service.CustomLabels); len(labels) > 0 {
spec.Container.Labels = labels
}
// Map x-caddy extension to spec.Caddy if specified.
if caddy, ok := service.Extensions[CaddyExtensionKey].(Caddy); ok && caddy.Config != "" {
spec.Caddy = &api.CaddySpec{
@@ -124,6 +128,10 @@ func ServiceSpecFromCompose(project *types.Project, serviceName string) (api.Ser
d := time.Duration(cfg.Monitor)
spec.UpdateConfig.MonitorPeriod = &d
}
if len(service.Deploy.Labels) > 0 {
spec.Labels = maps.Clone(service.Deploy.Labels)
}
}
// TODO: can service.tmpfs be handled as tmpfs volume mounts as well?
@@ -480,9 +488,6 @@ func validateServicesFeatures(project *types.Project) []error {
if service.DNSSearch != nil {
errs = append(errs, err(service.Name, "dns_search"))
}
if service.Labels != nil {
errs = append(errs, err(service.Name, "labels"))
}
if service.Links != nil {
errs = append(errs, err(service.Name, "links"))
}
@@ -517,9 +522,6 @@ func validateServicesFeatures(project *types.Project) []error {
if service.Deploy.RollbackConfig != nil {
errs = append(errs, err(service.Name, "deploy rollback_config"))
}
if len(service.Deploy.Labels) != 0 {
errs = append(errs, err(service.Name, "deploy labels"))
}
}
// we only allow the 'default' network, nothing else.
if x := service.Networks; x != nil {
+75
View File
@@ -875,6 +875,81 @@ services:
}
}
func TestServiceSpecFromCompose_Labels(t *testing.T) {
t.Parallel()
tests := []struct {
name string
composeYAML string
expectedContainerLabels map[string]string
expectedServiceLabels map[string]string
}{
{
name: "no labels",
composeYAML: `
services:
web:
image: nginx
`,
},
{
name: "container labels as list",
composeYAML: `
services:
web:
image: nginx
labels:
- com.example.team=web
- com.example.empty
`,
expectedContainerLabels: map[string]string{
"com.example.team": "web",
"com.example.empty": "",
},
},
{
name: "container and service labels as maps",
composeYAML: `
services:
web:
image: nginx
labels:
com.example.team: web
deploy:
labels:
tier: frontend
`,
expectedContainerLabels: map[string]string{"com.example.team": "web"},
expectedServiceLabels: map[string]string{"tier": "frontend"},
},
{
name: "service labels only",
composeYAML: `
services:
web:
image: nginx
deploy:
labels:
- tier=frontend
`,
expectedServiceLabels: map[string]string{"tier": "frontend"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
project, err := LoadProjectFromContent(context.Background(), tt.composeYAML)
require.NoError(t, err)
spec, err := ServiceSpecFromCompose(project, "web")
require.NoError(t, err)
assert.Equal(t, tt.expectedContainerLabels, spec.Container.Labels)
assert.Equal(t, tt.expectedServiceLabels, spec.Labels)
})
}
}
func TestServiceSpecFromCompose_UpdateConfig(t *testing.T) {
t.Parallel()
+33
View File
@@ -343,6 +343,39 @@ func (cli *Client) RemoveContainer(
return nil
}
// UpdateContainer updates the requested properties of the container within the service in place without recreating it.
func (cli *Client) UpdateContainer(
ctx context.Context, serviceNameOrID, containerNameOrID string, opts api.UpdateContainerOptions,
) error {
op, err := cli.resolveContainerOperation(ctx, serviceNameOrID, containerNameOrID)
if err != nil {
return err
}
req := &pb.UpdateServiceContainerRequest{
Id: op.containerID,
Cpu: opts.CPU,
Memory: opts.Memory,
MemoryReservation: opts.MemoryReservation,
}
if opts.ServiceLabels != nil {
req.ServiceLabels = &pb.Labels{Labels: *opts.ServiceLabels}
}
pw := progress.ContextWriter(op.ctx)
pw.Event(progress.NewEvent(op.eventID, progress.Working, "Updating"))
if _, err = cli.Docker.GRPCClient.UpdateServiceContainer(op.ctx, req); err != nil {
pw.Event(progress.ErrorEvent(op.eventID))
if status.Code(err) == codes.NotFound {
return api.ErrNotFound
}
return err
}
pw.Event(progress.NewEvent(op.eventID, progress.Done, "Updated"))
return nil
}
// ExecContainer executes a command in a container within the service.
// If containerNameOrID is empty, the first container in the service will be used.
func (cli *Client) ExecContainer(
+58 -13
View File
@@ -1,7 +1,7 @@
package deploy
import (
"reflect"
"maps"
"sort"
"github.com/google/go-cmp/cmp"
@@ -12,7 +12,8 @@ import (
type ContainerSpecStatus string
const (
ContainerUpToDate ContainerSpecStatus = "up-to-date"
ContainerUpToDate ContainerSpecStatus = "up-to-date"
// ContainerNeedsUpdate means the container can be updated in place without recreating it.
ContainerNeedsUpdate ContainerSpecStatus = "needs-update"
ContainerNeedsRecreate ContainerSpecStatus = "needs-recreate"
)
@@ -83,15 +84,7 @@ func EvalContainerSpecChange(current api.ServiceSpec, new api.ServiceSpec) Conta
}
}
// Device reservations and mappings are immutable, so we'll need to recreate if any have changed
if !reflect.DeepEqual(current.Container.Resources.DeviceReservations, newResources.DeviceReservations) {
return ContainerNeedsRecreate
}
if !reflect.DeepEqual(current.Container.Resources.Devices, newResources.Devices) {
return ContainerNeedsRecreate
}
// Ulimits are immutable, so we'll need to recreate if any have changed.
if !reflect.DeepEqual(current.Container.Resources.Ulimits, newResources.Ulimits) {
if current.Container.Resources.ValidateUpdateInPlace(newResources) != nil {
return ContainerNeedsRecreate
}
@@ -100,8 +93,10 @@ func EvalContainerSpecChange(current api.ServiceSpec, new api.ServiceSpec) Conta
return ContainerNeedsRecreate
}
// Remaining resources are mutable.
if !reflect.DeepEqual(current.Container.Resources, newResources) {
// CPU and memory limits are updated on the running container. Service labels are only stored in the service spec
// and not applied to Docker containers.
if !current.Container.Resources.Equals(newResources) ||
!cmp.Equal(current.Labels, new.Labels, cmpopts.EquateEmpty()) {
return ContainerNeedsUpdate
}
@@ -119,3 +114,53 @@ func sortConfigs(configs []api.ConfigSpec) {
return configs[i].Name < configs[j].Name
})
}
// EvalContainerChange evaluates whether the running container matches the new spec, can be updated in place to
// match it, or needs to be recreated. Unlike EvalContainerSpecChange, it also compares the new spec with the CPU and
// memory limits that Docker applies to the container, so the container is updated when they drifted from its spec.
func EvalContainerChange(current api.ServiceContainer, new api.ServiceSpec) ContainerSpecStatus {
status := EvalContainerSpecChange(current.ServiceSpec, new)
if status == ContainerNeedsRecreate {
return status
}
applied := current.AppliedResources()
newResources := new.Container.Resources
if applied.ValidateUpdateInPlace(newResources) != nil {
return ContainerNeedsRecreate
}
// Only CPU and memory limits can differ after the in-place update check above.
if !applied.Equals(newResources) {
return ContainerNeedsUpdate
}
return status
}
// ContainerUpdateOptions returns the container properties that can be updated in place and differ between the new
// spec and either the container spec or the limits Docker applies to the container. It should only be used when
// EvalContainerChange returns ContainerNeedsUpdate.
func ContainerUpdateOptions(current api.ServiceContainer, new api.ServiceSpec) api.UpdateContainerOptions {
var opts api.UpdateContainerOptions
stored, applied := current.ServiceSpec.Container.Resources, current.AppliedResources()
res := new.Container.Resources
if res.CPU != stored.CPU || res.CPU != applied.CPU {
opts.CPU = &res.CPU
}
if res.Memory != stored.Memory || res.Memory != applied.Memory {
opts.Memory = &res.Memory
}
if res.MemoryReservation != stored.MemoryReservation || res.MemoryReservation != applied.MemoryReservation {
opts.MemoryReservation = &res.MemoryReservation
}
if !cmp.Equal(current.ServiceSpec.Labels, new.Labels, cmpopts.EquateEmpty()) {
labels := maps.Clone(new.Labels)
if labels == nil {
labels = map[string]string{}
}
opts.ServiceLabels = &labels
}
return opts
}
+235 -4
View File
@@ -86,6 +86,223 @@ func TestEvalContainerSpecChange_ContainerTty(t *testing.T) {
assert.Equal(t, ContainerNeedsRecreate, EvalContainerSpecChange(newSpec, currentSpec))
}
func TestEvalContainerSpecChange_ContainerLabels(t *testing.T) {
t.Parallel()
currentSpec := api.ServiceSpec{
Container: api.ContainerSpec{
Image: "nginx:latest",
},
}
newSpec := api.ServiceSpec{
Container: api.ContainerSpec{
Image: "nginx:latest",
Labels: map[string]string{"com.example.team": "web"},
},
}
emptySpec := api.ServiceSpec{
Container: api.ContainerSpec{
Image: "nginx:latest",
Labels: map[string]string{},
},
}
assert.Equal(t, ContainerNeedsRecreate, EvalContainerSpecChange(currentSpec, newSpec))
assert.Equal(t, ContainerNeedsRecreate, EvalContainerSpecChange(newSpec, currentSpec))
assert.Equal(t, ContainerUpToDate, EvalContainerSpecChange(newSpec, newSpec))
assert.Equal(t, ContainerUpToDate, EvalContainerSpecChange(currentSpec, emptySpec))
}
func TestEvalContainerSpecChange_ServiceLabels(t *testing.T) {
t.Parallel()
currentSpec := api.ServiceSpec{
Container: api.ContainerSpec{
Image: "nginx:latest",
},
}
newSpec := api.ServiceSpec{
Container: api.ContainerSpec{
Image: "nginx:latest",
},
Labels: map[string]string{"tier": "frontend"},
}
emptySpec := api.ServiceSpec{
Container: api.ContainerSpec{
Image: "nginx:latest",
},
Labels: map[string]string{},
}
assert.Equal(t, ContainerNeedsUpdate, EvalContainerSpecChange(currentSpec, newSpec))
assert.Equal(t, ContainerNeedsUpdate, EvalContainerSpecChange(newSpec, currentSpec))
assert.Equal(t, ContainerUpToDate, EvalContainerSpecChange(newSpec, newSpec))
assert.Equal(t, ContainerUpToDate, EvalContainerSpecChange(currentSpec, emptySpec))
// Changes that require recreating the container take precedence over the in-place update.
newSpecWithImage := newSpec.Clone()
newSpecWithImage.Container.Image = "nginx:alpine"
assert.Equal(t, ContainerNeedsRecreate, EvalContainerSpecChange(currentSpec, newSpecWithImage))
}
func TestEvalContainerChange(t *testing.T) {
t.Parallel()
spec := api.ServiceSpec{
Container: api.ContainerSpec{
Image: "nginx:latest",
Resources: api.ContainerResources{CPU: api.Core, Memory: 256 * 1024 * 1024},
},
}
appliedSpec := container.Resources{NanoCPUs: api.Core, Memory: 256 * 1024 * 1024}
tests := []struct {
name string
hostConfig *container.HostConfig
change func(spec *api.ServiceSpec)
expected ContainerSpecStatus
}{
{
name: "applied limits match spec",
hostConfig: &container.HostConfig{Resources: appliedSpec},
change: func(*api.ServiceSpec) {},
expected: ContainerUpToDate,
},
{
name: "unknown host config falls back to spec limits",
change: func(*api.ServiceSpec) {},
expected: ContainerUpToDate,
},
{
name: "applied memory limit drifted from spec",
hostConfig: &container.HostConfig{Resources: container.Resources{NanoCPUs: api.Core, Memory: 512 * 1024 * 1024}},
change: func(*api.ServiceSpec) {},
expected: ContainerNeedsUpdate,
},
{
name: "applied limits already match new spec but stored spec doesn't",
hostConfig: &container.HostConfig{Resources: container.Resources{NanoCPUs: api.Core, Memory: 512 * 1024 * 1024}},
change: func(spec *api.ServiceSpec) { spec.Container.Resources.Memory = 512 * 1024 * 1024 },
expected: ContainerNeedsUpdate,
},
{
name: "applied limit not in spec can't be removed in place",
hostConfig: &container.HostConfig{Resources: container.Resources{
NanoCPUs: api.Core, Memory: 256 * 1024 * 1024, MemoryReservation: 128 * 1024 * 1024,
}},
change: func(*api.ServiceSpec) {},
expected: ContainerNeedsRecreate,
},
{
name: "spec change that requires recreating",
hostConfig: &container.HostConfig{Resources: appliedSpec},
change: func(spec *api.ServiceSpec) { spec.Container.Image = "nginx:alpine" },
expected: ContainerNeedsRecreate,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
newSpec := spec.Clone()
tt.change(&newSpec)
assert.Equal(t, tt.expected, EvalContainerChange(serviceContainer(spec, tt.hostConfig), newSpec))
})
}
}
func TestContainerUpdateOptions(t *testing.T) {
t.Parallel()
current := api.ServiceSpec{
Container: api.ContainerSpec{
Image: "nginx:latest",
Resources: api.ContainerResources{CPU: api.Core, Memory: 256 * 1024 * 1024},
},
Labels: map[string]string{"tier": "frontend"},
}
applied := &container.HostConfig{Resources: container.Resources{NanoCPUs: api.Core, Memory: 256 * 1024 * 1024}}
tests := []struct {
name string
hostConfig *container.HostConfig
change func(spec *api.ServiceSpec)
expected api.UpdateContainerOptions
}{
{
name: "no changes",
hostConfig: applied,
change: func(*api.ServiceSpec) {},
},
{
name: "CPU only",
hostConfig: applied,
change: func(spec *api.ServiceSpec) { spec.Container.Resources.CPU = 2 * api.Core },
expected: api.UpdateContainerOptions{CPU: new(int64(2 * api.Core))},
},
{
name: "memory and reservation",
hostConfig: applied,
change: func(spec *api.ServiceSpec) {
spec.Container.Resources.Memory = 512 * 1024 * 1024
spec.Container.Resources.MemoryReservation = 128 * 1024 * 1024
},
expected: api.UpdateContainerOptions{
Memory: new(int64(512 * 1024 * 1024)),
MemoryReservation: new(int64(128 * 1024 * 1024)),
},
},
{
name: "applied memory limit drifted from spec",
hostConfig: &container.HostConfig{Resources: container.Resources{
NanoCPUs: api.Core, Memory: 512 * 1024 * 1024,
}},
change: func(*api.ServiceSpec) {},
expected: api.UpdateContainerOptions{Memory: new(int64(256 * 1024 * 1024))},
},
{
name: "applied memory limit already matches new spec",
hostConfig: &container.HostConfig{Resources: container.Resources{
NanoCPUs: api.Core, Memory: 512 * 1024 * 1024,
}},
change: func(spec *api.ServiceSpec) { spec.Container.Resources.Memory = 512 * 1024 * 1024 },
expected: api.UpdateContainerOptions{Memory: new(int64(512 * 1024 * 1024))},
},
{
name: "change service labels",
hostConfig: applied,
change: func(spec *api.ServiceSpec) { spec.Labels = map[string]string{"tier": "backend"} },
expected: api.UpdateContainerOptions{ServiceLabels: &map[string]string{"tier": "backend"}},
},
{
name: "remove all service labels",
hostConfig: applied,
change: func(spec *api.ServiceSpec) { spec.Labels = nil },
expected: api.UpdateContainerOptions{ServiceLabels: &map[string]string{}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
newSpec := current.Clone()
tt.change(&newSpec)
assert.Equal(t, tt.expected, ContainerUpdateOptions(serviceContainer(current, tt.hostConfig), newSpec))
})
}
}
func serviceContainer(spec api.ServiceSpec, hostConfig *container.HostConfig) api.ServiceContainer {
return api.ServiceContainer{
Container: api.Container{InspectResponse: container.InspectResponse{
ContainerJSONBase: &container.ContainerJSONBase{HostConfig: hostConfig},
}},
ServiceSpec: spec,
}
}
func TestEvalContainerSpecChange_ContainerResources(t *testing.T) {
t.Parallel()
@@ -127,7 +344,7 @@ func TestEvalContainerSpecChange_ContainerResources(t *testing.T) {
CPU: 1000000000,
},
new: api.ContainerResources{},
want: ContainerNeedsUpdate,
want: ContainerNeedsRecreate,
},
// Memory
@@ -189,19 +406,33 @@ func TestEvalContainerSpecChange_ContainerResources(t *testing.T) {
name: "unset memory",
current: api.ContainerResources{Memory: 100 * 1024 * 1024, MemoryReservation: 50 * 1024 * 1024},
new: api.ContainerResources{MemoryReservation: 50 * 1024 * 1024},
want: ContainerNeedsUpdate,
want: ContainerNeedsRecreate,
},
{
name: "unset reservation",
current: api.ContainerResources{Memory: 100 * 1024 * 1024, MemoryReservation: 50 * 1024 * 1024},
new: api.ContainerResources{Memory: 100 * 1024 * 1024},
want: ContainerNeedsUpdate,
want: ContainerNeedsRecreate,
},
{
name: "unset memory and reservation",
current: api.ContainerResources{Memory: 100 * 1024 * 1024, MemoryReservation: 50 * 1024 * 1024},
new: api.ContainerResources{},
want: ContainerNeedsUpdate,
want: ContainerNeedsRecreate,
},
// Shared memory
{
name: "change shared memory",
current: api.ContainerResources{SharedMemory: 64 * 1024 * 1024},
new: api.ContainerResources{SharedMemory: 128 * 1024 * 1024},
want: ContainerNeedsRecreate,
},
{
name: "change memory and shared memory",
current: api.ContainerResources{Memory: 100 * 1024 * 1024},
new: api.ContainerResources{Memory: 200 * 1024 * 1024, SharedMemory: 64 * 1024 * 1024},
want: ContainerNeedsRecreate,
},
// Memory and CPU
+8 -1
View File
@@ -173,7 +173,7 @@ func (sp *ServicePlan) Format() string {
// FormatSummary counts operations in the service plan and renders a styled summary line.
func (sp *ServicePlan) FormatSummary() string {
var createCount, startFirstCount, stopFirstCount, removeCount int
var createCount, startFirstCount, stopFirstCount, updateCount, removeCount int
machines := make(map[string]struct{})
for _, op := range sp.Operations {
@@ -188,6 +188,9 @@ func (sp *ServicePlan) FormatSummary() string {
} else {
startFirstCount++
}
case *operation.UpdateContainerOperation:
machines[o.MachineID] = struct{}{}
updateCount++
case *operation.RemoveContainerOperation:
machines[o.MachineID] = struct{}{}
removeCount++
@@ -210,6 +213,10 @@ func (sp *ServicePlan) FormatSummary() string {
parts = append(parts,
tui.BoldYellow.Render(strconv.Itoa(stopFirstCount))+" "+tui.Yellow.Render("replace (stop-first)"))
}
if updateCount > 0 {
parts = append(parts,
tui.BoldYellow.Render(strconv.Itoa(updateCount))+" "+tui.Yellow.Render("update (in place)"))
}
if removeCount > 0 {
parts = append(parts,
tui.BoldRed.Render(strconv.Itoa(removeCount))+" "+tui.Red.Render("remove"))
+51
View File
@@ -0,0 +1,51 @@
package deploy
import (
"testing"
"github.com/charmbracelet/x/ansi"
"github.com/psviderski/uncloud/pkg/api"
"github.com/psviderski/uncloud/pkg/client/operation"
"github.com/stretchr/testify/assert"
)
func TestServicePlan_FormatSummary(t *testing.T) {
t.Parallel()
tests := []struct {
name string
ops []operation.Operation
expected string
}{
{
name: "in-place updates only",
ops: []operation.Operation{
&operation.UpdateContainerOperation{MachineID: "m-1"},
&operation.UpdateContainerOperation{MachineID: "m-2"},
},
expected: "2 update (in place) · across 2 machines",
},
{
name: "all operation types",
ops: []operation.Operation{
&operation.RunContainerOperation{MachineID: "m-1"},
&operation.ReplaceContainerOperation{MachineID: "m-1", Order: api.UpdateOrderStartFirst},
&operation.ReplaceContainerOperation{MachineID: "m-2", Order: api.UpdateOrderStopFirst},
&operation.UpdateContainerOperation{MachineID: "m-2"},
&operation.RemoveContainerOperation{MachineID: "m-3"},
},
expected: "1 create · 1 replace (start-first) · 1 replace (stop-first) · 1 update (in place) · 1 remove · " +
"across 3 machines",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
plan := ServicePlan{}
plan.Operations = tt.ops
assert.Equal(t, tt.expected, ansi.Strip(plan.FormatSummary()))
})
}
}
+50 -24
View File
@@ -92,7 +92,7 @@ func (s *RollingStrategy) planReplicated(svc *api.Service, spec api.ServiceSpec)
// Organise existing containers by machine.
containersOnMachine := make(map[string][]api.ServiceContainer)
upToDateContainersOnMachine := make(map[string]int)
reusableContainersOnMachine := make(map[string]int)
containerSpecStatuses := make(map[string]ContainerSpecStatus)
if svc != nil {
for _, c := range svc.Containers {
@@ -105,40 +105,50 @@ func (s *RollingStrategy) planReplicated(svc *api.Service, spec api.ServiceSpec)
if s.ForceRecreate {
status = ContainerNeedsRecreate
} else {
status = EvalContainerSpecChange(c.Container.ServiceSpec, spec)
status = EvalContainerChange(c.Container, spec)
}
containerSpecStatuses[c.Container.ID] = status
if status == ContainerUpToDate {
upToDateContainersOnMachine[c.MachineID] += 1
if status != ContainerNeedsRecreate {
reusableContainersOnMachine[c.MachineID] += 1
}
}
// Sort containers such that running containers with the desired spec are first.
// Sort containers such that running containers with the desired spec or that can be updated in place to
// the desired spec are first.
// TODO: consider ranking up-to-date containers above ones that need an in-place update (here, when sorting
// machines, and in reconcileGlobalContainer) to avoid updating a container while removing an up-to-date one.
isReusable := func(c api.MachineServiceContainer) bool {
// Not running containers have no status.
status, ok := containerSpecStatuses[c.Container.ID]
return ok && status != ContainerNeedsRecreate
}
slices.SortFunc(svc.Containers, func(c1, c2 api.MachineServiceContainer) int {
if status, ok := containerSpecStatuses[c1.Container.ID]; ok && status == ContainerUpToDate {
r1, r2 := isReusable(c1), isReusable(c2)
switch {
case r1 == r2:
return 0
case r1:
return -1
}
if status, ok := containerSpecStatuses[c2.Container.ID]; ok && status == ContainerUpToDate {
default:
return 1
}
return 0
})
for _, c := range svc.Containers {
containersOnMachine[c.MachineID] = append(containersOnMachine[c.MachineID], c.Container)
}
// Sort machines such that machines with the most up-to-date containers are first, followed by machines with
// existing containers, and finally machines without containers.
// Sort machines such that machines with the most containers that are up to date or can be updated in place
// are first, followed by machines with existing containers, and finally machines without containers.
slices.SortFunc(matchedMachines, func(m1, m2 *pb.MachineInfo) int {
if upToDateContainersOnMachine[m1.Id] > 0 && upToDateContainersOnMachine[m2.Id] > 0 {
return upToDateContainersOnMachine[m2.Id] - upToDateContainersOnMachine[m1.Id]
if reusableContainersOnMachine[m1.Id] > 0 && reusableContainersOnMachine[m2.Id] > 0 {
return reusableContainersOnMachine[m2.Id] - reusableContainersOnMachine[m1.Id]
}
if upToDateContainersOnMachine[m1.Id] > 0 {
if reusableContainersOnMachine[m1.Id] > 0 {
return -1
}
if upToDateContainersOnMachine[m2.Id] > 0 {
if reusableContainersOnMachine[m2.Id] > 0 {
return 1
}
return len(containersOnMachine[m2.Id]) - len(containersOnMachine[m1.Id])
@@ -170,7 +180,16 @@ func (s *RollingStrategy) planReplicated(svc *api.Service, spec api.ServiceSpec)
if status == ContainerUpToDate {
continue
}
// TODO: handle ContainerNeedsUpdate when update of mutable fields on a container is supported.
if status == ContainerNeedsUpdate {
plan.Operations = append(plan.Operations, &operation.UpdateContainerOperation{
ServiceID: plan.ServiceID,
MachineID: m.Id,
MachineName: m.Name,
Container: ctr,
Options: ContainerUpdateOptions(ctr, spec),
})
continue
}
}
// Replace the old container with a new one.
@@ -301,7 +320,7 @@ func reconcileGlobalContainer(
}
// Check if there is a container with the same spec already running. If so, remove the rest.
upToDate := false
reused := false
for i, c := range containers {
if !c.Container.State.Running || c.Container.State.Paused {
// Skip containers that are not running.
@@ -312,12 +331,21 @@ func reconcileGlobalContainer(
if forceRecreate {
status = ContainerNeedsRecreate
} else {
status = EvalContainerSpecChange(c.Container.ServiceSpec, spec)
status = EvalContainerChange(c.Container, spec)
}
if status == ContainerUpToDate {
// The container is already running with the same spec.
upToDate = true
if status != ContainerNeedsRecreate {
// The container is already running with the desired spec or can be updated in place to it.
reused = true
if status == ContainerNeedsUpdate {
ops = append(ops, &operation.UpdateContainerOperation{
ServiceID: serviceID,
MachineID: machine.Id,
MachineName: machine.Name,
Container: c.Container,
Options: ContainerUpdateOptions(c.Container, spec),
})
}
for j, old := range containers {
if i == j {
continue
@@ -331,10 +359,8 @@ func reconcileGlobalContainer(
}
break
}
// TODO: handle ContainerNeedsUpdate when update of mutable fields on a container is supported.
// Make sure to update preDeployOperation accordingly.
}
if upToDate {
if reused {
return ops, nil
}
+135 -1
View File
@@ -1,6 +1,8 @@
package deploy
import (
"slices"
"strings"
"testing"
"github.com/docker/docker/api/types/container"
@@ -11,6 +13,7 @@ import (
"github.com/psviderski/uncloud/pkg/client/deploy/scheduler"
"github.com/psviderski/uncloud/pkg/client/operation"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDetermineUpdateOrder(t *testing.T) {
@@ -424,6 +427,121 @@ func TestReconcileGlobalContainer(t *testing.T) {
}
}
func TestRollingStrategy_Plan_UpdateInPlace(t *testing.T) {
t.Parallel()
state := &scheduler.ClusterState{
Machines: []*scheduler.Machine{
{Info: &pb.MachineInfo{Id: "m-1", Name: "machine-1"}},
{Info: &pb.MachineInfo{Id: "m-2", Name: "machine-2"}},
},
}
newContainer := func(id string, spec api.ServiceSpec) api.ServiceContainer {
ctr := newServiceContainer(id, container.State{Running: true, Status: "running"})
ctr.ServiceSpec = spec
return ctr
}
tests := []struct {
name string
change func(spec *api.ServiceSpec)
// options are the expected in-place update options. Nil means the containers must be recreated.
options *api.UpdateContainerOptions
}{
{
name: "service labels",
change: func(spec *api.ServiceSpec) { spec.Labels = map[string]string{"tier": "backend"} },
options: &api.UpdateContainerOptions{ServiceLabels: &map[string]string{"tier": "backend"}},
},
{
name: "CPU and memory limits",
change: func(spec *api.ServiceSpec) {
spec.Container.Resources.CPU = 2 * api.Core
spec.Container.Resources.Memory = 512 * 1024 * 1024
},
options: &api.UpdateContainerOptions{
CPU: new(int64(2 * api.Core)),
Memory: new(int64(512 * 1024 * 1024)),
},
},
{
name: "remove memory limit",
change: func(spec *api.ServiceSpec) { spec.Container.Resources.Memory = 0 },
},
}
for _, tt := range tests {
for _, mode := range []string{api.ServiceModeReplicated, api.ServiceModeGlobal} {
t.Run(tt.name+"/"+mode, func(t *testing.T) {
currentSpec := api.ServiceSpec{
Name: "app",
Mode: mode,
Container: api.ContainerSpec{
Image: "nginx:latest",
Resources: api.ContainerResources{
CPU: api.Core,
Memory: 256 * 1024 * 1024,
},
},
Labels: map[string]string{"tier": "frontend"},
PreDeploy: &api.PreDeployHook{Command: []string{"db", "migrate"}},
}
if mode == api.ServiceModeReplicated {
currentSpec.Replicas = 2
}
newSpec := currentSpec.Clone()
tt.change(&newSpec)
ctr1 := newContainer("container-1", currentSpec)
ctr2 := newContainer("container-2", currentSpec)
svc := &api.Service{
ID: "svc-1",
Name: "app",
Mode: mode,
Containers: []api.MachineServiceContainer{
{MachineID: "m-1", MachineName: "machine-1", Container: ctr1},
{MachineID: "m-2", MachineName: "machine-2", Container: ctr2},
},
}
plan, err := (&RollingStrategy{}).Plan(state, svc, newSpec)
require.NoError(t, err)
if tt.options == nil {
require.NotEmpty(t, plan.Operations)
for _, op := range plan.Operations {
_, isUpdate := op.(*operation.UpdateContainerOperation)
assert.False(t, isUpdate, "Expected containers to be recreated, got %s", op)
}
return
}
// The replicated strategy shuffles machines so sort the operations to compare them deterministically.
ops := slices.Clone(plan.Operations)
slices.SortFunc(ops, func(a, b operation.Operation) int {
return strings.Compare(a.String(), b.String())
})
// The containers are updated in place and the pre-deploy hook is not run.
assertOperationsEqual(t, []operation.Operation{
&operation.UpdateContainerOperation{
ServiceID: "svc-1",
MachineID: "m-1",
MachineName: "machine-1",
Container: ctr1,
Options: *tt.options,
},
&operation.UpdateContainerOperation{
ServiceID: "svc-1",
MachineID: "m-2",
MachineName: "machine-2",
Container: ctr2,
Options: *tt.options,
},
}, ops)
})
}
}
}
func TestPreDeployOperations(t *testing.T) {
hook := &api.PreDeployHook{
Command: []string{"db", "migrate"},
@@ -502,6 +620,20 @@ func TestPreDeployOperations(t *testing.T) {
},
},
},
{
name: "plan has only UpdateContainerOperation",
plan: ServicePlan{
ServiceID: "svc-1",
ServiceName: "app",
Spec: api.ServiceSpec{PreDeploy: hook},
SequenceOperation: operation.SequenceOperation{
Operations: []operation.Operation{
&operation.UpdateContainerOperation{MachineID: "m-1", MachineName: "machine-1"},
},
},
},
expected: nil,
},
{
name: "plan has only RemoveContainerOperation",
plan: ServicePlan{
@@ -614,7 +746,8 @@ func TestPreDeployOperations(t *testing.T) {
},
svc: &api.Service{
HookContainers: []api.MachineServiceContainer{
{MachineID: "m-1", Container: newServiceContainer("stopped-hook", container.State{Status: "exited"})},
{MachineID: "m-1", Container: newServiceContainer("stopped-hook",
container.State{Status: "exited"})},
{MachineID: "m-1", Container: runningHook1},
{MachineID: "m-3", Container: runningHook2},
},
@@ -660,6 +793,7 @@ func newServiceContainer(id string, state container.State) api.ServiceContainer
ID: id,
State: &state,
},
Config: &container.Config{},
},
}}
}
+33
View File
@@ -285,6 +285,39 @@ func (o *ReplaceContainerOperation) String() string {
o.MachineID, o.ServiceID, o.OldContainer.ID, o.Order)
}
// UpdateContainerOperation updates the requested properties of a running container in place without recreating it.
type UpdateContainerOperation struct {
ServiceID string
MachineID string
// MachineName is used for formatting the operation as part of the deployment plan.
MachineName string
Container api.ServiceContainer
Options api.UpdateContainerOptions
}
func (o *UpdateContainerOperation) Execute(ctx context.Context, cli Client) error {
if err := cli.UpdateContainer(ctx, o.ServiceID, o.Container.ID, o.Options); err != nil {
return fmt.Errorf("update container: %w", err)
}
return nil
}
func (o *UpdateContainerOperation) Format() string {
displayName := o.Container.ServiceSpec.Name + tui.Faint.Render("/") + o.Container.ShortID()
return tui.BoldYellow.Render("~") + " " +
tui.Faint.Render("update container") + " " +
displayName + " " +
tui.Faint.Render("on") + " " +
o.MachineName + " " +
tui.Faint.Render("(in place)")
}
func (o *UpdateContainerOperation) String() string {
return fmt.Sprintf("UpdateContainerOperation[machine_id=%s service_id=%s container_id=%s]",
o.MachineID, o.ServiceID, o.Container.ID)
}
// stopOptions converts a stop grace period duration to Docker container stop options.
func stopOptions(gracePeriod *time.Duration) container.StopOptions {
if gracePeriod == nil {
+7
View File
@@ -123,6 +123,13 @@ func (c *recordingClient) RemoveContainer(
return nil
}
func (c *recordingClient) UpdateContainer(
_ context.Context, _, containerID string, _ api.UpdateContainerOptions,
) error {
c.calls = append(c.calls, "update:"+containerID)
return nil
}
func (c *recordingClient) CreatePreDeployHookContainer(
context.Context, string, api.ServiceSpec, string,
) (api.CreateContainerResponse, error) {