BREAKING CHANGE: resolve gRPC API proxy targets (machines) on server instead of client, needs upgrade to v0.20 (#247)

Co-authored-by: Pasha Sviderski <me@psviderski.name>
This commit is contained in:
Justin Bradford
2026-05-19 15:22:50 +10:00
committed by GitHub
co-authored by Pasha Sviderski
parent 03ff4cd51d
commit c95136eae6
29 changed files with 914 additions and 741 deletions
+17 -41
View File
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"os"
"strings"
"github.com/docker/cli/cli/streams"
"github.com/psviderski/uncloud/internal/machine/api/pb"
@@ -71,47 +70,24 @@ func (cli *Client) progressOut() *streams.Out {
return streams.NewOut(os.Stdout)
}
// proxyToMachine returns a new context that proxies gRPC requests to the specified machine.
func proxyToMachine(ctx context.Context, machine *pb.MachineInfo) context.Context {
machineIP, _ := machine.Network.ManagementIp.ToAddr()
md := metadata.Pairs("machines", machineIP.String())
// ProxyMachinesContext returns a new context that proxies gRPC requests to the specified machines.
// If namesOrIDs is nil or empty, all machines are included.
// This triggers One2Many proxying, which always injects metadata into the response.
func (cli *Client) ProxyMachinesContext(ctx context.Context, namesOrIDs []string) context.Context {
md := metadata.New(nil)
if len(namesOrIDs) == 0 {
md.Append("machines", "*")
} else {
md.Append("machines", namesOrIDs...)
}
return metadata.NewOutgoingContext(ctx, md)
}
// ProxyMachinesContext returns a new context that proxies gRPC requests to the specified machines.
// If namesOrIDs is nil, all machines are included.
func (cli *Client) ProxyMachinesContext(
ctx context.Context, namesOrIDs []string,
) (context.Context, api.MachineMembersList, error) {
// TODO: move the machine IP resolution to the proxy router to allow setting machine names and IDs in the metadata.
machines, err := cli.ListMachines(ctx, nil)
if err != nil {
return nil, nil, fmt.Errorf("list machines: %w", err)
}
var proxiedMachines api.MachineMembersList
var notFound []string
for _, nameOrID := range namesOrIDs {
if m := machines.FindByNameOrID(nameOrID); m != nil {
proxiedMachines = append(proxiedMachines, m)
} else {
notFound = append(notFound, nameOrID)
}
}
if len(notFound) > 0 {
return nil, nil, fmt.Errorf("machines not found: %s", strings.Join(notFound, ", "))
}
if len(namesOrIDs) == 0 {
proxiedMachines = machines
}
md := metadata.New(nil)
for _, m := range proxiedMachines {
machineIP, _ := m.Machine.Network.ManagementIp.ToAddr()
md.Append("machines", machineIP.String())
}
return metadata.NewOutgoingContext(ctx, md), proxiedMachines, nil
// ProxySingleMachineContext returns a new context that proxies gRPC requests to a single specified machine.
// This triggers One2One proxying, which does NOT inject metadata into the response.
// Use this for requests that expect a single response message without metadata wrapper.
func (cli *Client) ProxySingleMachineContext(ctx context.Context, nameOrID string) context.Context {
md := metadata.Pairs("machine", nameOrID)
return metadata.NewOutgoingContext(ctx, md)
}
+43 -42
View File
@@ -74,7 +74,7 @@ func (cli *Client) createServiceContainerWithPull(
resp.Name = containerName
// Proxy Docker gRPC requests to the selected machine.
ctx = proxyToMachine(ctx, machine.Machine)
ctx = cli.ProxySingleMachineContext(ctx, machine.Machine.Id)
pw := progress.ContextWriter(ctx)
eventID := cliprogress.NewContainerEventID(ctx, containerName, machine.Machine.Name)
@@ -259,27 +259,43 @@ func (cli *Client) InspectContainer(
return svc.FindContainer(containerNameOrID)
}
// StartContainer starts the specified container within the service.
func (cli *Client) StartContainer(ctx context.Context, serviceNameOrID, containerNameOrID string) error {
// containerOperationContext holds the context needed to perform an operation on a container.
type containerOperationContext struct {
ctx context.Context
containerID string
eventID string
}
// resolveContainerOperation resolves a container by name/ID and prepares the context for an operation.
func (cli *Client) resolveContainerOperation(
ctx context.Context, serviceNameOrID, containerNameOrID string,
) (containerOperationContext, error) {
ctr, err := cli.InspectContainer(ctx, serviceNameOrID, containerNameOrID)
if err != nil {
return err
return containerOperationContext{}, err
}
machine, err := cli.InspectMachine(ctx, ctr.MachineID)
eventID := cliprogress.ContainerEventID(ctx, ctr.Container.ServiceSpec.Name, ctr.Container.ID, ctr.MachineName)
return containerOperationContext{
ctx: cli.ProxySingleMachineContext(ctx, ctr.MachineID),
containerID: ctr.Container.ID,
eventID: eventID,
}, nil
}
// StartContainer starts the specified container within the service.
func (cli *Client) StartContainer(ctx context.Context, serviceNameOrID, containerNameOrID string) error {
op, err := cli.resolveContainerOperation(ctx, serviceNameOrID, containerNameOrID)
if err != nil {
return fmt.Errorf("inspect machine '%s': %w", ctr.MachineID, err)
}
ctx = proxyToMachine(ctx, machine.Machine)
pw := progress.ContextWriter(ctx)
eventID := cliprogress.ContainerEventID(ctx, ctr.Container.ServiceSpec.Name, ctr.Container.ID, machine.Machine.Name)
pw.Event(progress.StartingEvent(eventID))
if err = cli.Docker.StartContainer(ctx, ctr.Container.ID, container.StartOptions{}); err != nil {
return err
}
pw.Event(progress.StartedEvent(eventID))
pw := progress.ContextWriter(op.ctx)
pw.Event(progress.StartingEvent(op.eventID))
if err = cli.Docker.StartContainer(op.ctx, op.containerID, container.StartOptions{}); err != nil {
return err
}
pw.Event(progress.StartedEvent(op.eventID))
return nil
}
@@ -290,25 +306,17 @@ func (cli *Client) StartContainer(ctx context.Context, serviceNameOrID, containe
func (cli *Client) StopContainer(
ctx context.Context, serviceNameOrID, containerNameOrID string, opts container.StopOptions,
) error {
ctr, err := cli.InspectContainer(ctx, serviceNameOrID, containerNameOrID)
op, err := cli.resolveContainerOperation(ctx, serviceNameOrID, containerNameOrID)
if err != nil {
return err
}
machine, err := cli.InspectMachine(ctx, ctr.MachineID)
if err != nil {
return fmt.Errorf("inspect machine '%s': %w", ctr.MachineID, err)
}
ctx = proxyToMachine(ctx, machine.Machine)
pw := progress.ContextWriter(ctx)
eventID := cliprogress.ContainerEventID(ctx, ctr.Container.ServiceSpec.Name, ctr.Container.ID, machine.Machine.Name)
pw.Event(progress.StoppingEvent(eventID))
if err = cli.Docker.StopContainer(ctx, ctr.Container.ID, opts); err != nil {
pw := progress.ContextWriter(op.ctx)
pw.Event(progress.StoppingEvent(op.eventID))
if err = cli.Docker.StopContainer(op.ctx, op.containerID, opts); err != nil {
return err
}
pw.Event(progress.StoppedEvent(eventID))
pw.Event(progress.StoppedEvent(op.eventID))
return nil
}
@@ -319,25 +327,18 @@ func (cli *Client) StopContainer(
func (cli *Client) RemoveContainer(
ctx context.Context, serviceNameOrID, containerNameOrID string, opts container.RemoveOptions,
) error {
ctr, err := cli.InspectContainer(ctx, serviceNameOrID, containerNameOrID)
op, err := cli.resolveContainerOperation(ctx, serviceNameOrID, containerNameOrID)
if err != nil {
return err
}
machine, err := cli.InspectMachine(ctx, ctr.MachineID)
if err != nil {
return fmt.Errorf("inspect machine '%s': %w", ctr.MachineID, err)
}
ctx = proxyToMachine(ctx, machine.Machine)
pw := progress.ContextWriter(op.ctx)
pw := progress.ContextWriter(ctx)
eventID := cliprogress.ContainerEventID(ctx, ctr.Container.ServiceSpec.Name, ctr.Container.ID, machine.Machine.Name)
pw.Event(progress.RemovingEvent(eventID))
if err = cli.Docker.RemoveServiceContainer(ctx, ctr.Container.ID, opts); err != nil {
pw.Event(progress.RemovingEvent(op.eventID))
if err = cli.Docker.RemoveServiceContainer(op.ctx, op.containerID, opts); err != nil {
return err
}
pw.Event(progress.RemovedEvent(eventID))
pw.Event(progress.RemovedEvent(op.eventID))
return nil
}
@@ -374,7 +375,7 @@ func (cli *Client) ExecContainer(
}
// Proxy Docker gRPC requests to the machine hosting the container
ctx = proxyToMachine(ctx, machine.Machine)
ctx = cli.ProxySingleMachineContext(ctx, machine.Machine.Id)
// Execute the command in the container
exitCode, err := cli.Docker.ExecContainer(ctx, machinedocker.ExecConfig{
@@ -451,7 +452,7 @@ func (cli *Client) WaitContainerHealthy(
}
// For containers with a health check, wait until Docker reports healthy or unhealthy.
mctx := proxyToMachine(ctx, machine.Machine)
mctx := cli.ProxySingleMachineContext(ctx, machine.Machine.Id)
mctx, cancel := context.WithTimeout(mctx, healthcheckTimeout(mc.Container.Config.Healthcheck))
defer cancel()
ticker := time.NewTicker(1 * time.Second)
+27 -24
View File
@@ -25,6 +25,7 @@ import (
"github.com/docker/docker/pkg/jsonmessage"
"github.com/docker/go-connections/nat"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/psviderski/uncloud/internal/cli/tui"
"github.com/psviderski/uncloud/internal/docker"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/constants"
@@ -58,10 +59,7 @@ func (cli *Client) InspectRemoteImage(ctx context.Context, id string) ([]api.Mac
// it lists images on all machines.
func (cli *Client) ListImages(ctx context.Context, filter api.ImageFilter) ([]api.MachineImages, error) {
// Broadcast the image list request to the specified machines or all machines if none specified.
listCtx, machines, err := cli.ProxyMachinesContext(ctx, filter.Machines)
if err != nil {
return nil, fmt.Errorf("create request context to broadcast to machines: %w", err)
}
listCtx := cli.ProxyMachinesContext(ctx, filter.Machines)
opts := image.ListOptions{Manifests: true}
if filter.Name != "" {
@@ -80,32 +78,36 @@ func (cli *Client) ListImages(ctx context.Context, filter api.ImageFilter) ([]ap
return nil, err
}
machineImages := make([]api.MachineImages, len(resp.Messages))
for i, msg := range resp.Messages {
machineImages[i].Metadata = msg.Metadata
// TODO: handle this in the grpc-proxy router and always provide Metadata if possible.
machineImages := make([]api.MachineImages, 0, len(resp.Messages))
for _, msg := range resp.Messages {
// NOTE: Metadata should never be nil in practice. This is legacy fallback that will be removed.
if msg.Metadata == nil {
// Metadata can be nil if the request was broadcasted to only one machine.
machineImages[i].Metadata = &pb.Metadata{
Machine: machines[0].Machine.Id,
}
} else {
// Replace management IP with machine ID for friendlier error messages.
// TODO: migrate Metadata.Machine to use machine ID instead of IP in the grpc-proxy router.
if m := machines.FindByManagementIP(msg.Metadata.Machine); m != nil {
machineImages[i].Metadata.Machine = m.Machine.Id
}
if msg.Metadata.Error != "" {
continue
}
tui.PrintWarning("metadata is missing in response from unknown server")
continue
}
if msg.Metadata.Error != "" {
// Continue processing other messages even if some machines return an error to avoid a partial failure
// of the entire command.
tui.PrintWarning(fmt.Sprintf(
"failed to list images on machine %s: %s", msg.Metadata.MachineName, msg.Metadata.Error,
))
continue
}
mi := api.MachineImages{
Metadata: msg.Metadata,
ContainerdStore: msg.ContainerdStore,
}
if len(msg.Images) > 0 {
if err = json.Unmarshal(msg.Images, &machineImages[i].Images); err != nil {
if err = json.Unmarshal(msg.Images, &mi.Images); err != nil {
return nil, fmt.Errorf("unmarshal images: %w", err)
}
}
machineImages[i].ContainerdStore = msg.ContainerdStore
machineImages = append(machineImages, mi)
}
return machineImages, nil
@@ -350,7 +352,8 @@ func (cli *Client) pushImageToMachine(
pw.Event(progress.NewEvent(proxyEventID, progress.Error, err.Error()))
return fmt.Errorf("run socat container with unix socket to proxy unregistry: %w", err)
}
slog.Debug("Started unix socket socat proxy container.", "id", proxyCtrID, "hostPort", proxyPort, "socket", socketPath)
slog.Debug("Started unix socket socat proxy container.",
"id", proxyCtrID, "hostPort", proxyPort, "socket", socketPath)
}
pw.Event(progress.Event{
+2 -8
View File
@@ -102,10 +102,7 @@ func (cli *Client) ServiceLogs(
func (cli *Client) ContainerLogs(
ctx context.Context, machineNameOrID string, containerID string, opts api.ServiceLogsOptions,
) (<-chan api.LogEntry, error) {
proxyCtx, _, err := cli.ProxyMachinesContext(ctx, []string{machineNameOrID})
if err != nil {
return nil, fmt.Errorf("create request context to proxy to machine '%s': %w", machineNameOrID, err)
}
proxyCtx := cli.ProxySingleMachineContext(ctx, machineNameOrID)
req := &pb.LogsRequest{
Id: containerID,
@@ -201,10 +198,7 @@ func (cli *Client) MachineLogs(
func (cli *Client) systemdServiceLogs(
ctx context.Context, machineID, unit string, opts api.ServiceLogsOptions,
) (<-chan api.LogEntry, error) {
proxyCtx, _, err := cli.ProxyMachinesContext(ctx, []string{machineID})
if err != nil {
return nil, fmt.Errorf("create request context to proxy to machine '%s': %w", machineID, err)
}
proxyCtx := cli.ProxySingleMachineContext(ctx, machineID)
req := &pb.LogsRequest{
Id: unit,
+29 -46
View File
@@ -4,12 +4,12 @@ import (
"context"
"errors"
"fmt"
"os"
"slices"
"sync"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/volume"
"github.com/psviderski/uncloud/internal/cli/tui"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/pkg/api"
"github.com/psviderski/uncloud/pkg/client/deploy/scheduler"
@@ -96,16 +96,14 @@ func (cli *Client) InspectService(ctx context.Context, nameOrID string) (api.Ser
}
// Broadcast the container list request to all available machines.
machineIDByManagementIP := make(map[string]string)
md := metadata.New(nil)
for _, m := range machines {
if m.State == pb.MachineMember_UP || m.State == pb.MachineMember_SUSPECT {
machineIP, _ := m.Machine.Network.ManagementIp.ToAddr()
md.Append("machines", machineIP.String())
machineIDByManagementIP[machineIP.String()] = m.Machine.Id
md.Append("machines", m.Machine.Id)
} else {
tui.PrintWarning(fmt.Sprintf("failed to list service containers on machine '%s' (state is %s). "+
"The results may be incomplete.", m.Machine.Name, m.State.String()))
}
// TODO: warning about machines that are DOWN.
}
listCtx := metadata.NewOutgoingContext(ctx, md)
@@ -120,37 +118,25 @@ func (cli *Client) InspectService(ctx context.Context, nameOrID string) (api.Ser
foundByID := false
var containers []api.MachineServiceContainer
for _, mc := range machineContainers {
// Metadata can be nil if the request was broadcasted to only one machine.
if mc.Metadata == nil && len(machineContainers) > 1 {
return svc, errors.New("something went wrong with gRPC proxy: metadata is missing for a machine response")
}
if mc.Metadata != nil && mc.Metadata.Error != "" {
// TODO: return failed machines in the response.
fmt.Printf("WARNING: failed to list containers on machine '%s': %s\n",
mc.Metadata.Machine, mc.Metadata.Error)
// NOTE: Metadata should never be nil in practice. This is legacy fallback that will be removed.
if mc.Metadata == nil {
tui.PrintWarning("metadata is missing in response from unknown server")
continue
}
machineID := ""
if mc.Metadata == nil {
// ListServiceContainers was proxied to only one machine.
for _, v := range machineIDByManagementIP {
machineID = v
break
}
} else {
var ok bool
machineID, ok = machineIDByManagementIP[mc.Metadata.Machine]
if !ok {
return svc, fmt.Errorf("machine name not found for management IP: %s", mc.Metadata.Machine)
}
if mc.Metadata.Error != "" {
// TODO: return failed machines in the response.
tui.PrintWarning(fmt.Sprintf("failed to list containers on machine '%s': %s",
mc.Metadata.MachineName, mc.Metadata.Error))
continue
}
// Collect both regular and hook containers for the service.
for _, ctr := range append(mc.Containers, mc.HookContainers...) {
containers = append(containers, api.MachineServiceContainer{
MachineID: machineID,
Container: ctr,
MachineID: mc.Metadata.MachineId,
MachineName: mc.Metadata.MachineName,
Container: ctr,
})
if ctr.ServiceID() == nameOrID {
@@ -232,16 +218,6 @@ func (cli *Client) RemoveService(ctx context.Context, id string) error {
return err
}
machines, err := cli.ListMachines(ctx, nil)
if err != nil {
return fmt.Errorf("list machines: %w", err)
}
machineManagementIPByID := make(map[string]string)
for _, m := range machines {
machineIP, _ := m.Machine.Network.ManagementIp.ToAddr()
machineManagementIPByID[m.Machine.Id] = machineIP.String()
}
wg := sync.WaitGroup{}
errCh := make(chan error)
@@ -353,10 +329,11 @@ func (cli *Client) ListServices(ctx context.Context) ([]api.Service, error) {
md := metadata.New(nil)
for _, m := range machines {
if m.State == pb.MachineMember_UP || m.State == pb.MachineMember_SUSPECT {
machineIP, _ := m.Machine.Network.ManagementIp.ToAddr()
md.Append("machines", machineIP.String())
md.Append("machines", m.Machine.Id)
} else {
tui.PrintWarning(fmt.Sprintf("failed to list service containers on machine '%s' (state is %s). "+
"The results may be incomplete.", m.Machine.Name, m.State.String()))
}
// TODO: warning about machines that are DOWN.
}
listCtx := metadata.NewOutgoingContext(ctx, md)
@@ -371,10 +348,16 @@ func (cli *Client) ListServices(ctx context.Context) ([]api.Service, error) {
// Most of the code can be reused in both InspectService and ListServices.
servicesByID := make(map[string]api.Service)
for _, mc := range machineContainers {
if mc.Metadata != nil && mc.Metadata.Error != "" {
// NOTE: Metadata should never be nil in practice. This is legacy fallback that will be removed.
if mc.Metadata == nil {
tui.PrintWarning("metadata is missing in response from unknown server")
continue
}
if mc.Metadata.Error != "" {
// TODO: return failed machines in the response.
fmt.Fprintf(os.Stderr, "WARNING: failed to list containers on machine '%s': %s\n",
mc.Metadata.Machine, mc.Metadata.Error)
tui.PrintWarning(fmt.Sprintf("failed to list containers on machine '%s': %s",
mc.Metadata.MachineName, mc.Metadata.Error))
continue
}
+12 -23
View File
@@ -9,7 +9,6 @@ import (
"github.com/docker/docker/api/types/volume"
cliprogress "github.com/psviderski/uncloud/internal/cli/progress"
"github.com/psviderski/uncloud/internal/cli/tui"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/pkg/api"
)
@@ -28,7 +27,7 @@ func (cli *Client) CreateVolume(
return resp, fmt.Errorf("inspect machine '%s': %w", machineNameOrID, err)
}
// Proxy Docker gRPC requests to the selected machine.
ctx = proxyToMachine(ctx, machine.Machine)
ctx = cli.ProxySingleMachineContext(ctx, machine.Machine.Id)
pw := progress.ContextWriter(ctx)
eventID := cliprogress.VolumeEventID(opts.Name, machine.Machine.Name)
@@ -57,11 +56,7 @@ func (cli *Client) ListVolumes(ctx context.Context, filter *api.VolumeFilter) ([
proxyMachines = filter.Machines
}
listCtx, machines, err := cli.ProxyMachinesContext(ctx, proxyMachines)
if err != nil {
return nil, fmt.Errorf("create request context to broadcast to all machines: %w", err)
}
listCtx := cli.ProxyMachinesContext(ctx, proxyMachines)
machineVolumes, err := cli.Docker.ListVolumes(listCtx, volume.ListOptions{})
if err != nil {
return nil, err
@@ -70,28 +65,22 @@ func (cli *Client) ListVolumes(ctx context.Context, filter *api.VolumeFilter) ([
var volumes []api.MachineVolume
// Process responses from all machines.
for _, mv := range machineVolumes {
if mv.Metadata != nil && mv.Metadata.Error != "" {
// TODO: return failed machines in the response.
tui.PrintWarning(fmt.Sprintf("failed to list volumes on machine '%s': %s",
mv.Metadata.Machine, mv.Metadata.Error))
if mv.Metadata == nil {
tui.PrintWarning("metadata is missing in response from unknown server")
continue
}
var m *pb.MachineMember
if mv.Metadata == nil {
// ListVolumes was proxied to only one machine.
m = machines[0]
} else {
m = machines.FindByManagementIP(mv.Metadata.Machine)
if m == nil {
return nil, fmt.Errorf("machine not found by management IP: %s", mv.Metadata.Machine)
}
if mv.Metadata.Error != "" {
// TODO: return failed machines in the response.
tui.PrintWarning(fmt.Sprintf("failed to list volumes on machine '%s': %s", mv.Metadata.MachineName,
mv.Metadata.Error))
continue
}
for _, vol := range mv.Response.Volumes {
volumes = append(volumes, api.MachineVolume{
MachineID: m.Machine.Id,
MachineName: m.Machine.Name,
MachineID: mv.Metadata.MachineId,
MachineName: mv.Metadata.MachineName,
Volume: *vol,
})
}
@@ -118,7 +107,7 @@ func (cli *Client) RemoveVolume(ctx context.Context, machineNameOrID, volumeName
return fmt.Errorf("inspect machine '%s': %w", machineNameOrID, err)
}
// Proxy Docker gRPC requests to the selected machine.
ctx = proxyToMachine(ctx, machine.Machine)
ctx = cli.ProxySingleMachineContext(ctx, machine.Machine.Id)
pw := progress.ContextWriter(ctx)
eventID := cliprogress.VolumeEventID(volumeName, machine.Machine.Name)