chore: use original service spec instead of deriving from container, fixes diff for scale

This commit is contained in:
Pavel Sviderski
2025-03-30 12:54:36 +10:00
parent 0975cbab66
commit 64f6a4f3d3
22 changed files with 956 additions and 405 deletions
+6 -5
View File
@@ -3,10 +3,11 @@ package api
import (
"context"
"fmt"
"slices"
"github.com/docker/docker/api/types/container"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"google.golang.org/grpc/metadata"
"slices"
)
type Client interface {
@@ -21,10 +22,10 @@ type ContainerClient interface {
CreateContainer(
ctx context.Context, serviceID string, spec ServiceSpec, machineID string,
) (container.CreateResponse, error)
InspectContainer(ctx context.Context, serviceID, containerID string) (MachineContainer, error)
RemoveContainer(ctx context.Context, serviceID, containerID string, opts container.RemoveOptions) error
StartContainer(ctx context.Context, serviceID, containerID string) error
StopContainer(ctx context.Context, serviceID, containerID string, opts container.StopOptions) error
InspectContainer(ctx context.Context, serviceNameOrID, containerNameOrID string) (MachineServiceContainer, error)
RemoveContainer(ctx context.Context, serviceNameOrID, containerNameOrID string, opts container.RemoveOptions) error
StartContainer(ctx context.Context, serviceNameOrID, containerNameOrID string) error
StopContainer(ctx context.Context, serviceNameOrID, containerNameOrID string, opts container.StopOptions) error
}
type DNSClient interface {
+68 -70
View File
@@ -1,11 +1,13 @@
package api
import (
"encoding/json"
"fmt"
"github.com/docker/docker/api/types"
"github.com/docker/go-units"
"strings"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/go-units"
)
const (
@@ -21,73 +23,6 @@ type Container struct {
types.ContainerJSON
}
// NameWithoutSlash returns the container name without the leading slash.
// TODO: modify Name in original ContainerJSON structure when inspecting a Docker container and get rid of this method.
func (c *Container) NameWithoutSlash() string {
return c.Name[1:]
}
// ServiceID returns the ID of the service this container belongs to.
func (c *Container) ServiceID() string {
return c.Config.Labels[LabelServiceID]
}
// ServiceName returns the name of the service this container belongs to.
func (c *Container) ServiceName() string {
return c.Config.Labels[LabelServiceName]
}
// ServiceMode returns the replication mode of the service this container belongs to.
func (c *Container) ServiceMode() string {
return c.Config.Labels[LabelServiceMode]
}
// ServicePorts returns the ports this container publishes as part of its service.
func (c *Container) ServicePorts() ([]PortSpec, error) {
encoded, ok := c.Config.Labels[LabelServicePorts]
if !ok {
return nil, nil
}
if strings.TrimSpace(encoded) == "" {
return nil, nil
}
publishPorts := strings.Split(encoded, ",")
ports := make([]PortSpec, len(publishPorts))
for i, p := range publishPorts {
port, err := ParsePortSpec(strings.TrimSpace(p))
if err != nil {
return nil, err
}
ports[i] = port
}
return ports, nil
}
// ServiceSpec constructs a service spec from the container's configuration.
func (c *Container) ServiceSpec() (ServiceSpec, error) {
ports, err := c.ServicePorts()
if err != nil {
return ServiceSpec{}, fmt.Errorf("get service ports: %w", err)
}
// TODO: many properties on the container such as Config.Cmd or Config.Entrypoint are populated when the container
// is created. Figure out how to get a spec that is equal to the initial spec.
return ServiceSpec{
Container: ContainerSpec{
Command: c.Config.Cmd,
Entrypoint: c.Config.Entrypoint,
Image: c.Config.Image,
Init: c.HostConfig.Init,
Volumes: c.HostConfig.Binds,
},
Mode: c.ServiceMode(),
Name: c.ServiceName(),
Ports: ports,
}, nil
}
// Healthy determines if the container is running and healthy.
// A running container with no health check configured is considered healthy.
func (c *Container) Healthy() bool {
@@ -156,8 +91,71 @@ func (c *Container) HumanState() (string, error) {
c.State.ExitCode, units.HumanDuration(time.Now().UTC().Sub(finishedAt))), nil
}
func (c *Container) UnmarshalJSON(data []byte) error {
// A temporary type that's identical to Container but doesn't have the UnmarshalJSON method.
type ContainerAlias Container
var temp ContainerAlias
if err := json.Unmarshal(data, &temp); err != nil {
return err
}
*c = Container(temp)
c.Name = strings.TrimPrefix(c.Name, "/")
return nil
}
type ServiceContainer struct {
Container
ServiceSpec ServiceSpec
}
type MachineContainer struct {
MachineID string
Container Container
}
// ServiceID returns the ID of the service this container belongs to.
func (c *ServiceContainer) ServiceID() string {
return c.Config.Labels[LabelServiceID]
}
// ServiceName returns the name of the service this container belongs to.
func (c *ServiceContainer) ServiceName() string {
return c.Config.Labels[LabelServiceName]
}
// ServiceMode returns the replication mode of the service this container belongs to.
func (c *ServiceContainer) ServiceMode() string {
return c.Config.Labels[LabelServiceMode]
}
// ServicePorts returns the ports this container publishes as part of its service.
func (c *ServiceContainer) ServicePorts() ([]PortSpec, error) {
encoded, ok := c.Config.Labels[LabelServicePorts]
if !ok {
return nil, nil
}
if strings.TrimSpace(encoded) == "" {
return nil, nil
}
publishPorts := strings.Split(encoded, ",")
ports := make([]PortSpec, len(publishPorts))
for i, p := range publishPorts {
port, err := ParsePortSpec(strings.TrimSpace(p))
if err != nil {
return nil, err
}
ports[i] = port
}
return ports, nil
}
// ConflictingServicePorts returns a list of service ports that conflict with the given ports.
func (c *Container) ConflictingServicePorts(ports []PortSpec) ([]PortSpec, error) {
func (c *ServiceContainer) ConflictingServicePorts(ports []PortSpec) ([]PortSpec, error) {
svcPorts, err := c.ServicePorts()
if err != nil {
return nil, fmt.Errorf("get service ports: %w", err)
+5 -50
View File
@@ -1,60 +1,15 @@
package api
import (
"net/netip"
"testing"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/netip"
"reflect"
"testing"
)
func TestContainer_ServiceSpec(t *testing.T) {
t.Parallel()
init := true
ctr := &Container{ContainerJSON: types.ContainerJSON{
ContainerJSONBase: &types.ContainerJSONBase{
HostConfig: &container.HostConfig{
Binds: []string{"/host/path:/container/path"},
Init: &init,
},
},
Config: &container.Config{
Cmd: []string{"/app/server"},
Image: "app:latest",
Labels: map[string]string{
LabelServiceID: "test-service-id",
LabelServiceName: "test-service-name",
LabelServicePorts: "app.example.com:8000/https",
},
},
}}
expectedSpec := ServiceSpec{
Container: ContainerSpec{
Command: []string{"/app/server"},
Image: "app:latest",
Init: &init,
Volumes: []string{"/host/path:/container/path"},
},
Name: "test-service-name",
Ports: []PortSpec{
{
Hostname: "app.example.com",
ContainerPort: 8000,
Protocol: ProtocolHTTPS,
Mode: PortModeIngress,
},
},
}
spec, err := ctr.ServiceSpec()
require.NoError(t, err)
assert.True(t, reflect.DeepEqual(spec, expectedSpec))
}
func TestContainer_Healthy(t *testing.T) {
t.Parallel()
@@ -325,13 +280,13 @@ func TestContainer_ConflictingServicePorts(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctr := &Container{ContainerJSON: types.ContainerJSON{
ctr := &ServiceContainer{Container: Container{ContainerJSON: types.ContainerJSON{
Config: &container.Config{
Labels: map[string]string{
LabelServicePorts: tt.containerPorts,
},
},
}}
}}}
got, err := ctr.ConflictingServicePorts(tt.checkPorts)
if tt.wantErr {
+11 -10
View File
@@ -5,12 +5,13 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"github.com/distribution/reference"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"maps"
"reflect"
"regexp"
"slices"
"github.com/distribution/reference"
"github.com/psviderski/uncloud/internal/machine/api/pb"
)
const (
@@ -206,12 +207,12 @@ type Service struct {
ID string
Name string
Mode string
Containers []MachineContainer
Containers []MachineServiceContainer
}
type MachineContainer struct {
type MachineServiceContainer struct {
MachineID string
Container Container
Container ServiceContainer
}
// Endpoints returns the exposed HTTP and HTTPS endpoints of the service.
@@ -261,7 +262,7 @@ func (s *Service) Endpoints() []string {
func ServiceFromProto(s *pb.Service) (Service, error) {
var err error
containers := make([]MachineContainer, len(s.Containers))
containers := make([]MachineServiceContainer, len(s.Containers))
for i, sc := range s.Containers {
containers[i], err = machineContainerFromProto(sc)
if err != nil {
@@ -277,14 +278,14 @@ func ServiceFromProto(s *pb.Service) (Service, error) {
}, nil
}
func machineContainerFromProto(sc *pb.Service_Container) (MachineContainer, error) {
func machineContainerFromProto(sc *pb.Service_Container) (MachineServiceContainer, error) {
var c Container
if err := json.Unmarshal(sc.Container, &c); err != nil {
return MachineContainer{}, fmt.Errorf("unmarshal container: %w", err)
return MachineServiceContainer{}, fmt.Errorf("unmarshal container: %w", err)
}
return MachineContainer{
return MachineServiceContainer{
MachineID: sc.MachineId,
Container: c,
Container: ServiceContainer{Container: c},
}, nil
}
+2 -1
View File
@@ -4,13 +4,14 @@ import (
"context"
"errors"
"fmt"
"os"
"github.com/docker/cli/cli/streams"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/docker"
"github.com/psviderski/uncloud/pkg/api"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"os"
)
// Client is a client for the machine API.
+17 -20
View File
@@ -190,29 +190,26 @@ func toPullProgressEvent(jm jsonmessage.JSONMessage) *progress.Event {
}
// InspectContainer returns the information about the specified container within the service.
func (cli *Client) InspectContainer(ctx context.Context, serviceID, containerID string) (api.MachineContainer, error) {
var ctr api.MachineContainer
svc, err := cli.InspectService(ctx, serviceID)
func (cli *Client) InspectContainer(
ctx context.Context, serviceNameOrID, containerNameOrID string,
) (api.MachineServiceContainer, error) {
svc, err := cli.InspectService(ctx, serviceNameOrID)
if err != nil {
return ctr, fmt.Errorf("inspect service: %w", err)
return api.MachineServiceContainer{}, fmt.Errorf("inspect service: %w", err)
}
for _, c := range svc.Containers {
if c.Container.ID == containerID || c.Container.NameWithoutSlash() == containerID {
ctr = c
if c.Container.ID == containerNameOrID || c.Container.Name == containerNameOrID {
return c, nil
}
}
if ctr.MachineID == "" {
return ctr, api.ErrNotFound
}
return ctr, nil
return api.MachineServiceContainer{}, api.ErrNotFound
}
// StartContainer starts the specified container within the service.
func (cli *Client) StartContainer(ctx context.Context, serviceID, containerID string) error {
ctr, err := cli.InspectContainer(ctx, serviceID, containerID)
func (cli *Client) StartContainer(ctx context.Context, serviceNameOrID, containerNameOrID string) error {
ctr, err := cli.InspectContainer(ctx, serviceNameOrID, containerNameOrID)
if err != nil {
return err
}
@@ -224,7 +221,7 @@ func (cli *Client) StartContainer(ctx context.Context, serviceID, containerID st
ctx = proxyToMachine(ctx, machine.Machine)
pw := progress.ContextWriter(ctx)
eventID := fmt.Sprintf("Container %s on %s", ctr.Container.NameWithoutSlash(), machine.Machine.Name)
eventID := fmt.Sprintf("Container %s on %s", ctr.Container.Name, machine.Machine.Name)
pw.Event(progress.StartingEvent(eventID))
if err = cli.Docker.StartContainer(ctx, ctr.Container.ID, container.StartOptions{}); err != nil {
@@ -237,9 +234,9 @@ func (cli *Client) StartContainer(ctx context.Context, serviceID, containerID st
// StopContainer stops the specified container within the service.
func (cli *Client) StopContainer(
ctx context.Context, serviceID, containerID string, opts container.StopOptions,
ctx context.Context, serviceNameOrID, containerNameOrID string, opts container.StopOptions,
) error {
ctr, err := cli.InspectContainer(ctx, serviceID, containerID)
ctr, err := cli.InspectContainer(ctx, serviceNameOrID, containerNameOrID)
if err != nil {
return err
}
@@ -251,7 +248,7 @@ func (cli *Client) StopContainer(
ctx = proxyToMachine(ctx, machine.Machine)
pw := progress.ContextWriter(ctx)
eventID := fmt.Sprintf("Container %s on %s", ctr.Container.NameWithoutSlash(), machine.Machine.Name)
eventID := fmt.Sprintf("Container %s on %s", ctr.Container.Name, machine.Machine.Name)
pw.Event(progress.StoppingEvent(eventID))
if err = cli.Docker.StopContainer(ctx, ctr.Container.ID, opts); err != nil {
@@ -264,9 +261,9 @@ func (cli *Client) StopContainer(
// RemoveContainer removes the specified container within the service.
func (cli *Client) RemoveContainer(
ctx context.Context, serviceID, containerNameOrID string, opts container.RemoveOptions,
ctx context.Context, serviceNameOrID, containerNameOrID string, opts container.RemoveOptions,
) error {
ctr, err := cli.InspectContainer(ctx, serviceID, containerNameOrID)
ctr, err := cli.InspectContainer(ctx, serviceNameOrID, containerNameOrID)
if err != nil {
return err
}
@@ -278,7 +275,7 @@ func (cli *Client) RemoveContainer(
ctx = proxyToMachine(ctx, machine.Machine)
pw := progress.ContextWriter(ctx)
eventID := fmt.Sprintf("Container %s on %s", ctr.Container.NameWithoutSlash(), machine.Machine.Name)
eventID := fmt.Sprintf("Container %s on %s", ctr.Container.Name, machine.Machine.Name)
pw.Event(progress.RemovingEvent(eventID))
if err = cli.Docker.RemoveServiceContainer(ctx, ctr.Container.ID, opts); err != nil {
+3 -1
View File
@@ -2,6 +2,7 @@ package deploy
import (
"fmt"
"github.com/psviderski/uncloud/pkg/api"
)
@@ -11,7 +12,8 @@ const ContainerUpToDate ContainerSpecStatus = "up-to-date"
const ContainerNeedsUpdate ContainerSpecStatus = "needs-update"
const ContainerNeedsRecreate ContainerSpecStatus = "needs-recreate"
func CompareContainerToSpec(ctr api.Container, spec api.ServiceSpec) (ContainerSpecStatus, error) {
func CompareContainerToSpec(ctr api.ServiceContainer, spec api.ServiceSpec) (ContainerSpecStatus, error) {
// TODO: replace the hash comparison with a more detailed comparison of ctr.ServiceSpec and spec.
specHash, err := spec.ImmutableHash()
if err != nil {
return "", fmt.Errorf("calculate immutable hash for service spec: %w", err)
+7 -6
View File
@@ -3,11 +3,12 @@ package deploy
import (
"context"
"fmt"
"math/rand/v2"
"slices"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/secret"
"github.com/psviderski/uncloud/pkg/api"
"math/rand/v2"
"slices"
)
// Strategy defines how a service should be deployed or updated. Different implementations can provide various
@@ -89,7 +90,7 @@ func (s *RollingStrategy) planReplicated(
})
// Organise existing containers by machine.
containersOnMachine := make(map[string][]api.Container)
containersOnMachine := make(map[string][]api.ServiceContainer)
upToDateContainersOnMachine := make(map[string]int)
containerSpecStatuses := make(map[string]ContainerSpecStatus)
if svc != nil {
@@ -111,7 +112,7 @@ func (s *RollingStrategy) planReplicated(
}
// Sort containers such that running containers with the desired spec are first.
slices.SortFunc(svc.Containers, func(c1, c2 api.MachineContainer) int {
slices.SortFunc(svc.Containers, func(c1, c2 api.MachineServiceContainer) int {
if status, ok := containerSpecStatuses[c1.Container.ID]; ok && status == ContainerUpToDate {
return -1
}
@@ -222,7 +223,7 @@ func (s *RollingStrategy) planGlobal(
// Map machineID to service containers on that machine. For the global mode, there should be at most one
// container per machine but we use a slice to handle multiple containers that may exist due to a bug
// or interruption in the previous deployment.
containersOnMachine := make(map[string][]api.MachineContainer)
containersOnMachine := make(map[string][]api.MachineServiceContainer)
if svc != nil {
for _, c := range svc.Containers {
containersOnMachine[c.MachineID] = append(containersOnMachine[c.MachineID], c)
@@ -271,7 +272,7 @@ func (s *RollingStrategy) planGlobal(
// It ensures exactly one container with the desired spec is running on the machine by creating a new container and
// removing old ones. If there is a host port conflict, it stops the old container before starting a new one.
func reconcileGlobalContainer(
containers []api.MachineContainer, spec api.ServiceSpec, serviceID, machineID string,
containers []api.MachineServiceContainer, spec api.ServiceSpec, serviceID, machineID string,
) ([]Operation, error) {
var ops []Operation
+2 -1
View File
@@ -3,6 +3,7 @@ package client
import (
"context"
"fmt"
"github.com/psviderski/uncloud/pkg/api"
)
@@ -46,7 +47,7 @@ func (cli *Client) ServiceOperationNameResolver(ctx context.Context, svc api.Ser
}
containerNames := make(map[string]string, len(svc.Containers))
for _, c := range svc.Containers {
containerNames[c.Container.ID] = c.Container.NameWithoutSlash()
containerNames[c.Container.ID] = c.Container.Name
}
return NewNameResolver(machineNames, containerNames), nil
+22 -35
View File
@@ -4,17 +4,18 @@ import (
"context"
"errors"
"fmt"
"os"
"slices"
"sync"
"github.com/docker/compose/v2/pkg/progress"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/pkg/api"
"github.com/psviderski/uncloud/pkg/client/deploy"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"slices"
"sync"
)
type RunServiceResponse struct {
@@ -64,8 +65,8 @@ func (cli *Client) RunService(
}
// InspectService returns detailed information about a service and its containers.
// The id parameter can be either a service ID or name.
func (cli *Client) InspectService(ctx context.Context, id string) (api.Service, error) {
// The nameOrID parameter can be either a service name or ID.
func (cli *Client) InspectService(ctx context.Context, nameOrID string) (api.Service, error) {
var svc api.Service
machines, err := cli.ListMachines(ctx)
@@ -87,22 +88,16 @@ func (cli *Client) InspectService(ctx context.Context, id string) (api.Service,
}
listCtx := metadata.NewOutgoingContext(ctx, md)
// List only uncloud-managed containers that belong to some service.
opts := container.ListOptions{
All: true,
Filters: filters.NewArgs(
filters.Arg("label", api.LabelServiceID),
filters.Arg("label", api.LabelManaged),
),
}
machineContainers, err := cli.Docker.ListContainers(listCtx, opts)
// List all service containers including stopped ones.
opts := container.ListOptions{All: true}
machineContainers, err := cli.Docker.ListServiceContainers(listCtx, nameOrID, opts)
if err != nil {
return svc, fmt.Errorf("list containers: %w", err)
}
// Collect all containers on all machines that belong to the specified service.
foundByID := false
var containers []api.MachineContainer
var containers []api.MachineServiceContainer
for _, mc := range machineContainers {
// Metadata can be nil if the request was broadcasted to only one machine.
if mc.Metadata == nil && len(machineContainers) > 1 {
@@ -130,15 +125,14 @@ func (cli *Client) InspectService(ctx context.Context, id string) (api.Service,
}
}
for _, c := range mc.Containers {
ctr := api.Container{ContainerJSON: c}
if ctr.ServiceID() == id || ctr.ServiceName() == id {
containers = append(containers, api.MachineContainer{
for _, ctr := range mc.Containers {
if ctr.ServiceID() == nameOrID || ctr.ServiceName() == nameOrID {
containers = append(containers, api.MachineServiceContainer{
MachineID: machineID,
Container: ctr,
})
if ctr.ServiceID() == id {
if ctr.ServiceID() == nameOrID {
foundByID = true
}
}
@@ -153,15 +147,15 @@ func (cli *Client) InspectService(ctx context.Context, id string) (api.Service,
// may not prevent this), or a service name might match another service's ID. In these cases, matching by ID takes
// priority over matching by name.
if foundByID {
containers = slices.DeleteFunc(containers, func(mc api.MachineContainer) bool {
return mc.Container.ServiceID() != id
containers = slices.DeleteFunc(containers, func(mc api.MachineServiceContainer) bool {
return mc.Container.ServiceID() != nameOrID
})
} else {
// Matched only by name but there could be multiple services with the same name.
serviceID := containers[0].Container.ServiceID()
for _, mc := range containers[1:] {
if mc.Container.ServiceID() != serviceID {
return svc, fmt.Errorf("multiple services found with name '%s', use the service ID instead", id)
return svc, fmt.Errorf("multiple services found with name '%s', use the service ID instead", nameOrID)
}
}
}
@@ -273,15 +267,9 @@ func (cli *Client) ListServices(ctx context.Context) ([]api.Service, error) {
}
listCtx := metadata.NewOutgoingContext(ctx, md)
// List only uncloud-managed containers that belong to some service.
opts := container.ListOptions{
All: true,
Filters: filters.NewArgs(
filters.Arg("label", api.LabelServiceID),
filters.Arg("label", api.LabelManaged),
),
}
machineContainers, err := cli.Docker.ListContainers(listCtx, opts)
// List all containers including stopped ones.
opts := container.ListOptions{All: true}
machineContainers, err := cli.Docker.ListServiceContainers(listCtx, "", opts)
if err != nil {
return nil, fmt.Errorf("list containers: %w", err)
}
@@ -292,13 +280,12 @@ func (cli *Client) ListServices(ctx context.Context) ([]api.Service, error) {
for _, mc := range machineContainers {
if mc.Metadata != nil && mc.Metadata.Error != "" {
// TODO: return failed machines in the response.
fmt.Printf("WARNING: failed to list containers on machine '%s': %s\n",
fmt.Fprintf(os.Stderr, "WARNING: failed to list containers on machine '%s': %s\n",
mc.Metadata.Machine, mc.Metadata.Error)
continue
}
for _, c := range mc.Containers {
ctr := api.Container{ContainerJSON: c}
for _, ctr := range mc.Containers {
if _, ok := servicesByID[ctr.ServiceID()]; ok {
continue
}