feat(images): implement RemoveImage gRPC API with extra protection for used images

.mockery.yml

@@ -1,4 +1,10 @@
 packages:
+  github.com/docker/docker/client:
+    config:
+      dir: internal/machine/docker
+      pkgname: docker
+    interfaces:
+      APIClient:
   github.com/psviderski/uncloud/internal/machine/caddyconfig:
     interfaces:
       CaddyfileValidator:

HACKING.md

@@ -116,7 +116,7 @@ mise run proto
 Regenerate mocks:
 
 ```shell
-make mocks
+mise run mocks
 ```
 
 ## Uncloud in Docker (UCinD)

Makefile

@@ -21,10 +21,6 @@ corrosion-multiarch-image-push:
 ucind-multiarch-image-push:
 	docker buildx build --push --platform linux/amd64,linux/arm64 -t "$(UCIND_IMAGE)" --target ucind .
 
-.PHONY: mocks
-mocks:
-	@mockery
-
 .PHONY: test
 test:
 ifeq ($(TEST_NAME),)

api/pb/docker.proto

@@ -24,6 +24,7 @@ service Docker {
   // Docker auth credentials if necessary.
   rpc InspectRemoteImage(InspectRemoteImageRequest) returns (InspectRemoteImageResponse);
   rpc ListImages(ListImagesRequest) returns (ListImagesResponse);
+  rpc RemoveImage(RemoveImageRequest) returns (RemoveImageResponse);
 
   rpc CreateVolume(CreateVolumeRequest) returns (CreateVolumeResponse);
   rpc ListVolumes(ListVolumesRequest) returns (ListVolumesResponse);
@@ -179,6 +180,40 @@ message ListImagesRequest {
   bytes options = 1;
 }
 
+message RemoveImageRequest {
+  string id = 1;
+  bool force = 2;
+  bool prune_children = 3;
+  repeated ImagePlatform platforms = 4;
+}
+
+// ImagePlatform describes an OCI image platform.
+message ImagePlatform {
+  string architecture = 1;
+  string os = 2;
+  string os_version = 3;
+  repeated string os_features = 4;
+  string variant = 5;
+}
+
+message RemoveImageResponse {
+  // Must contain only one repeated messages field to allow broadcasting RemoveImage requests to multiple machines.
+  repeated MachineImageRemoval messages = 1;
+}
+
+message MachineImageRemoval {
+  Metadata metadata = 1;
+  repeated ImageRemoval results = 2;
+}
+
+// ImageRemoval reports an image deletion or an untagged image reference.
+message ImageRemoval {
+  // The image ID of an image that was deleted.
+  string deleted = 1;
+  // The image reference of an image that was untagged.
+  string untagged = 2;
+}
+
 message ListImagesResponse {
   // Must contain only one repeated messages field to allow broadcasting ListImages requests to multiple machines.
   repeated MachineImages messages = 1;

api/pb/docker_grpc.pb.go

@@ -32,6 +32,7 @@ const (
 	Docker_InspectImage_FullMethodName            = "/api.Docker/InspectImage"
 	Docker_InspectRemoteImage_FullMethodName      = "/api.Docker/InspectRemoteImage"
 	Docker_ListImages_FullMethodName              = "/api.Docker/ListImages"
+	Docker_RemoveImage_FullMethodName             = "/api.Docker/RemoveImage"
 	Docker_CreateVolume_FullMethodName            = "/api.Docker/CreateVolume"
 	Docker_ListVolumes_FullMethodName             = "/api.Docker/ListVolumes"
 	Docker_RemoveVolume_FullMethodName            = "/api.Docker/RemoveVolume"
@@ -59,6 +60,7 @@ type DockerClient interface {
 	// Docker auth credentials if necessary.
 	InspectRemoteImage(ctx context.Context, in *InspectRemoteImageRequest, opts ...grpc.CallOption) (*InspectRemoteImageResponse, error)
 	ListImages(ctx context.Context, in *ListImagesRequest, opts ...grpc.CallOption) (*ListImagesResponse, error)
+	RemoveImage(ctx context.Context, in *RemoveImageRequest, opts ...grpc.CallOption) (*RemoveImageResponse, error)
 	CreateVolume(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error)
 	ListVolumes(ctx context.Context, in *ListVolumesRequest, opts ...grpc.CallOption) (*ListVolumesResponse, error)
 	RemoveVolume(ctx context.Context, in *RemoveVolumeRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
@@ -217,6 +219,16 @@ func (c *dockerClient) ListImages(ctx context.Context, in *ListImagesRequest, op
 	return out, nil
 }
 
+func (c *dockerClient) RemoveImage(ctx context.Context, in *RemoveImageRequest, opts ...grpc.CallOption) (*RemoveImageResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(RemoveImageResponse)
+	err := c.cc.Invoke(ctx, Docker_RemoveImage_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *dockerClient) CreateVolume(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error) {
 	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
 	out := new(CreateVolumeResponse)
@@ -305,6 +317,7 @@ type DockerServer interface {
 	// Docker auth credentials if necessary.
 	InspectRemoteImage(context.Context, *InspectRemoteImageRequest) (*InspectRemoteImageResponse, error)
 	ListImages(context.Context, *ListImagesRequest) (*ListImagesResponse, error)
+	RemoveImage(context.Context, *RemoveImageRequest) (*RemoveImageResponse, error)
 	CreateVolume(context.Context, *CreateVolumeRequest) (*CreateVolumeResponse, error)
 	ListVolumes(context.Context, *ListVolumesRequest) (*ListVolumesResponse, error)
 	RemoveVolume(context.Context, *RemoveVolumeRequest) (*emptypb.Empty, error)
@@ -358,6 +371,9 @@ func (UnimplementedDockerServer) InspectRemoteImage(context.Context, *InspectRem
 func (UnimplementedDockerServer) ListImages(context.Context, *ListImagesRequest) (*ListImagesResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method ListImages not implemented")
 }
+func (UnimplementedDockerServer) RemoveImage(context.Context, *RemoveImageRequest) (*RemoveImageResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method RemoveImage not implemented")
+}
 func (UnimplementedDockerServer) CreateVolume(context.Context, *CreateVolumeRequest) (*CreateVolumeResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method CreateVolume not implemented")
 }
@@ -591,6 +607,24 @@ func _Docker_ListImages_Handler(srv interface{}, ctx context.Context, dec func(i
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Docker_RemoveImage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(RemoveImageRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(DockerServer).RemoveImage(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: Docker_RemoveImage_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(DockerServer).RemoveImage(ctx, req.(*RemoveImageRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _Docker_CreateVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(CreateVolumeRequest)
 	if err := dec(in); err != nil {
@@ -760,6 +794,10 @@ var Docker_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "ListImages",
 			Handler:    _Docker_ListImages_Handler,
 		},
+		{
+			MethodName: "RemoveImage",
+			Handler:    _Docker_RemoveImage_Handler,
+		},
 		{
 			MethodName: "CreateVolume",
 			Handler:    _Docker_CreateVolume_Handler,

internal/machine/caddyconfig/mocks_test.go

@@ -16,10 +16,19 @@ func NewMockCaddyfileValidator(t interface {
 	mock.TestingT
 	Cleanup(func())
 }) *MockCaddyfileValidator {
+	if helper, ok := t.(interface{ Helper() }); ok {
+		helper.Helper()
+	}
+
 	mock := &MockCaddyfileValidator{}
 	mock.Mock.Test(t)
 
-	t.Cleanup(func() { mock.AssertExpectations(t) })
+	t.Cleanup(func() {
+		if helper, ok := t.(interface{ Helper() }); ok {
+			helper.Helper()
+		}
+		mock.AssertExpectations(t)
+	})
 
 	return mock
 }

internal/machine/docker/controller.go

@@ -27,7 +27,7 @@ const (
 type Controller struct {
 	// machineID is the ID of the machine where the managed Docker daemon is running.
 	machineID string
-	client    *client.Client
+	client    client.APIClient
 	service   *Service
 	store     *store.Store
 }

internal/machine/docker/image.go

@@ -0,0 +1,79 @@
+package docker
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/containerd/errdefs"
+	"github.com/distribution/reference"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/image"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/psviderski/uncloud/pkg/api"
+)
+
+// RemoveImage removes an image while protecting names used by containers, even when force is requested.
+func (s *Service) RemoveImage(ctx context.Context, id string, opts image.RemoveOptions) ([]image.DeleteResponse, error) {
+	if err := s.checkImageRemovalConflicts(ctx, id); err != nil {
+		return nil, err
+	}
+	// Docker enforces its own conflicts, including protection of a used image's final reference.
+	return s.Client.ImageRemove(ctx, id, opts)
+}
+
+// checkImageRemovalConflicts rejects an image ID used by a running or stopped container, or a name used by a
+// container while it still resolves to that container's image. A digest request also checks the repository tags
+// Docker would remove with it. Unused aliases are allowed.
+func (s *Service) checkImageRemovalConflicts(ctx context.Context, id string) error {
+	img, err := s.Client.ImageInspect(ctx, id)
+	if err != nil {
+		return err
+	}
+	// Match Docker's resolved ID before parsing names, since a short ID also parses as a repository name.
+	isID := strings.HasPrefix(strings.TrimPrefix(img.ID, "sha256:"), strings.TrimPrefix(id, "sha256:"))
+	var named reference.Named
+	if !isID {
+		named, err = reference.ParseDockerRef(id)
+		if err != nil {
+			return errdefs.ErrInvalidArgument.WithMessage(err.Error())
+		}
+	}
+	_, isDigest := named.(reference.Canonical)
+
+	containers, err := s.Client.ContainerList(ctx, container.ListOptions{
+		All: true, Filters: filters.NewArgs(filters.Arg("ancestor", img.ID)),
+	})
+	if err != nil {
+		return fmt.Errorf("list containers using image: %w", err)
+	}
+
+	for _, ctr := range containers {
+		// The ancestor filter can include descendants. A retagged name must also be checked against its current ID.
+		if ctr.ImageID != img.ID {
+			continue
+		}
+
+		usedRef, err := reference.ParseDockerRef(ctr.Image)
+		// Familiar names make "alpine:latest" and "docker.io/library/alpine:latest" identify the same tag.
+		removesUsedName := err == nil && named != nil && reference.FamiliarString(usedRef) == reference.FamiliarString(named)
+		if isDigest && err == nil && usedRef.Name() == named.Name() {
+			// Removing a digest also removes this repository's tags pointing at the same image.
+			removesUsedName = true
+		}
+		if isID || removesUsedName {
+			ctrID, state := stringid.TruncateID(ctr.ID), "stopped"
+			if serviceName := ctr.Labels[api.LabelServiceName]; serviceName != "" {
+				ctrID = serviceName + "/" + ctrID
+			}
+			if ctr.State == container.StateRunning || ctr.State == container.StatePaused {
+				state = container.StateRunning
+			}
+			return errdefs.ErrConflict.WithMessage(
+				fmt.Sprintf("image '%s' is used by container '%s' (%s)", id, ctrID, state))
+		}
+	}
+
+	return nil
+}

internal/machine/docker/image_test.go

@@ -0,0 +1,175 @@
+package docker
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/containerd/errdefs"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/image"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCheckImageRemovalConflicts(t *testing.T) {
+	imageID := "sha256:" + strings.Repeat("a", 64)
+	otherID := "sha256:" + strings.Repeat("b", 64)
+	digestRef := "myapp@" + imageID
+
+	tests := []struct {
+		name             string
+		requested        string
+		configured       string
+		containerImageID string
+		serviceName      string
+		blocked          bool
+	}{
+		{name: "used tag", requested: "myapp:current", configured: "myapp:current", serviceName: "web", blocked: true},
+		{name: "qualified request", requested: "docker.io/library/myapp:current", configured: "myapp:current", blocked: true},
+		{name: "qualified container reference", requested: "myapp:current", configured: "docker.io/library/myapp:current", blocked: true},
+		{name: "implicit latest", requested: "myapp", configured: "myapp:latest", blocked: true},
+		{name: "unused alias", requested: "myapp:old", configured: "myapp:current"},
+		{name: "unused alias of ID-used image", requested: "myapp:old", configured: imageID},
+		{name: "tag may clean up a used digest", requested: "myapp:old", configured: digestRef},
+		{name: "used digest", requested: digestRef, configured: digestRef, blocked: true},
+		{name: "digest also removes used tag", requested: digestRef, configured: "myapp:current", blocked: true},
+		{name: "unused repository digest", requested: digestRef, configured: "other:keep"},
+		{name: "image ID", requested: imageID, configured: "myapp:current", blocked: true},
+		{name: "short image ID", requested: "aaaaaaaa", configured: "myapp:current", blocked: true},
+		{name: "short image ID with algorithm", requested: "sha256:aaaaaaaa", configured: "myapp:current", blocked: true},
+		{name: "retagged name", requested: "myapp:current", configured: "myapp:current", containerImageID: otherID},
+	}
+
+	for _, tt := range tests {
+		for _, state := range []string{container.StateExited, container.StateRunning} {
+			t.Run(fmt.Sprintf("%s/%s", tt.name, state), func(t *testing.T) {
+				containerImageID := tt.containerImageID
+				if containerImageID == "" {
+					containerImageID = imageID
+				}
+
+				cli := NewMockAPIClient(t)
+				cli.EXPECT().ImageInspect(mock.Anything, tt.requested).
+					Return(image.InspectResponse{ID: imageID}, nil).Once()
+				cli.EXPECT().ContainerList(mock.Anything, mock.MatchedBy(func(opts container.ListOptions) bool {
+					return opts.All // Stopped containers must be checked too.
+				})).Return([]container.Summary{{
+					ID:      strings.Repeat("c", 64),
+					Labels:  map[string]string{api.LabelServiceName: tt.serviceName},
+					Names:   []string{"/web"},
+					ImageID: containerImageID,
+					Image:   tt.configured,
+					State:   state,
+				}}, nil).Once()
+
+				service := NewService(cli, nil)
+				err := service.checkImageRemovalConflicts(context.Background(), tt.requested)
+
+				if tt.blocked {
+					require.True(t, errdefs.IsConflict(err), "%v", err)
+					assert.ErrorContains(t, err, tt.requested)
+					if tt.serviceName != "" {
+						assert.ErrorContains(t, err, "container 'web/cccccccccccc'")
+					} else {
+						assert.ErrorContains(t, err, "container 'cccccccccccc'")
+					}
+					wantState := "stopped"
+					if state == container.StateRunning {
+						wantState = state
+					}
+					assert.ErrorContains(t, err, wantState)
+				} else {
+					require.NoError(t, err)
+				}
+			})
+		}
+	}
+}
+
+func TestCheckImageRemovalConflicts_PropagatesLookupErrors(t *testing.T) {
+	imageID := "sha256:" + strings.Repeat("a", 64)
+	t.Run("image lookup fails", func(t *testing.T) {
+		cli := NewMockAPIClient(t)
+		lookupErr := errdefs.ErrNotFound.WithMessage("no such image")
+		cli.EXPECT().ImageInspect(mock.Anything, "myapp:old").Return(image.InspectResponse{}, lookupErr).Once()
+
+		service := NewService(cli, nil)
+		require.ErrorIs(t, service.checkImageRemovalConflicts(context.Background(), "myapp:old"), lookupErr)
+	})
+
+	t.Run("container lookup fails", func(t *testing.T) {
+		cli := NewMockAPIClient(t)
+		lookupErr := errors.New("Docker unavailable")
+		cli.EXPECT().ImageInspect(mock.Anything, "myapp:old").Return(image.InspectResponse{ID: imageID}, nil).Once()
+		cli.EXPECT().ContainerList(mock.Anything, mock.Anything).Return(nil, lookupErr).Once()
+
+		service := NewService(cli, nil)
+		err := service.checkImageRemovalConflicts(context.Background(), "myapp:old")
+
+		require.ErrorIs(t, err, lookupErr)
+		assert.ErrorContains(t, err, "list containers using image")
+	})
+}
+
+func TestServiceRemoveImage_RejectsUsedNameBeforeDeletion(t *testing.T) {
+	imageID := "sha256:" + strings.Repeat("a", 64)
+	cli := NewMockAPIClient(t)
+	cli.EXPECT().ImageInspect(mock.Anything, "myapp:current").Return(image.InspectResponse{ID: imageID}, nil).Once()
+	cli.EXPECT().ContainerList(mock.Anything, mock.Anything).Return([]container.Summary{{
+		ID: "container-id", ImageID: imageID, Image: "myapp:current", State: container.StateRunning,
+	}}, nil).Once()
+
+	service := NewService(cli, nil)
+	// ImageRemove has no expectation: any deletion call would fail this test.
+	results, err := service.RemoveImage(context.Background(), "myapp:current", image.RemoveOptions{Force: true})
+
+	require.True(t, errdefs.IsConflict(err), "%v", err)
+	assert.Nil(t, results)
+}
+
+func TestServiceRemoveImage_DelegatesToDocker(t *testing.T) {
+	imageID := "sha256:" + strings.Repeat("a", 64)
+	tests := []struct {
+		name    string
+		opts    image.RemoveOptions
+		results []image.DeleteResponse
+		err     error
+	}{
+		{
+			name:    "options and results pass through",
+			opts:    image.RemoveOptions{Force: true, PruneChildren: true, Platforms: []ocispec.Platform{{OS: "linux", Architecture: "amd64"}}},
+			results: []image.DeleteResponse{{Deleted: imageID}},
+		},
+		{
+			name: "Docker rejects final reference removal",
+			opts: image.RemoveOptions{PruneChildren: true},
+			err:  errdefs.ErrConflict.WithMessage("container is using its referenced image"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cli := NewMockAPIClient(t)
+			cli.EXPECT().ImageInspect(mock.Anything, "myapp:old").Return(image.InspectResponse{ID: imageID}, nil).Once()
+			cli.EXPECT().ContainerList(mock.Anything, mock.Anything).Return([]container.Summary{{
+				ID: "container-id", ImageID: imageID, Image: "myapp@" + imageID, State: container.StateRunning,
+			}}, nil).Once()
+			cli.EXPECT().ImageRemove(mock.Anything, "myapp:old", tt.opts).Return(tt.results, tt.err).Once()
+
+			service := NewService(cli, nil)
+			results, err := service.RemoveImage(context.Background(), "myapp:old", tt.opts)
+			if tt.err != nil {
+				require.ErrorIs(t, err, tt.err)
+			} else {
+				require.NoError(t, err)
+			}
+			assert.Equal(t, tt.results, results)
+		})
+	}
+}

internal/machine/docker/server.go

@@ -56,7 +56,7 @@ var fullDockerIDRegex = regexp.MustCompile(`^[a-f0-9]{64}$`)
 // Server implements the gRPC Docker service that proxies requests to the Docker daemon.
 type Server struct {
 	pb.UnimplementedDockerServer
-	client  *client.Client
+	client  client.APIClient
 	service *Service
 	db      *sqlx.DB
 	// internalDNSIP is a function that returns the IP address of the internal DNS server. It may return an empty
@@ -349,6 +349,41 @@ func (s *Server) InspectImage(ctx context.Context, req *pb.InspectImageRequest)
 	}, nil
 }
 
+// RemoveImage removes an image while protecting names used by containers, even when force is requested.
+func (s *Server) RemoveImage(ctx context.Context, req *pb.RemoveImageRequest) (*pb.RemoveImageResponse, error) {
+	opts := image.RemoveOptions{Force: req.Force, PruneChildren: req.PruneChildren}
+	for _, platform := range req.Platforms {
+		opts.Platforms = append(opts.Platforms, ocispec.Platform{
+			Architecture: platform.Architecture,
+			OS:           platform.Os,
+			OSVersion:    platform.OsVersion,
+			OSFeatures:   platform.OsFeatures,
+			Variant:      platform.Variant,
+		})
+	}
+
+	removals, err := s.service.RemoveImage(ctx, req.Id, opts)
+	if err != nil {
+		switch {
+		case errdefs.IsNotFound(err):
+			return nil, status.Error(codes.NotFound, err.Error())
+		case errdefs.IsConflict(err):
+			return nil, status.Error(codes.FailedPrecondition, err.Error())
+		case errdefs.IsInvalidArgument(err):
+			return nil, status.Error(codes.InvalidArgument, err.Error())
+		default:
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	}
+
+	result := &pb.MachineImageRemoval{}
+	for _, removal := range removals {
+		result.Results = append(result.Results, &pb.ImageRemoval{Deleted: removal.Deleted, Untagged: removal.Untagged})
+	}
+
+	return &pb.RemoveImageResponse{Messages: []*pb.MachineImageRemoval{result}}, nil
+}
+
 // InspectRemoteImage returns the image metadata for an image in a remote registry using the machine's Docker auth
 // credentials if necessary.
 func (s *Server) InspectRemoteImage(

internal/machine/docker/service.go

@@ -31,13 +31,13 @@ import (
 // from the machine database.
 type Service struct {
 	// Client is a Docker client for managing Docker resources.
-	Client *client.Client
+	Client client.APIClient
 	// db is a connection to the machine database.
 	db *sqlx.DB
 }
 
 // NewService creates a new Docker service instance.
-func NewService(client *client.Client, db *sqlx.DB) *Service {
+func NewService(client client.APIClient, db *sqlx.DB) *Service {
 	return &Service{
 		Client: client,
 		db:     db,

mise.lock

@@ -1,36 +1,43 @@
 # @generated - this file is auto-generated by `mise lock` https://mise.jdx.dev/dev-tools/mise-lock.html
 
 [[tools."aqua:vektra/mockery"]]
-version = "3.5.3"
+version = "3.8.1"
 backend = "aqua:vektra/mockery"
 
 [tools."aqua:vektra/mockery"."platforms.linux-arm64"]
-checksum = "sha256:ec02ca72b7ad1ff5a1de5503a27d3b1b3f230d722eb35c5b92e43e295a8e1290"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Linux_arm64.tar.gz"
+checksum = "sha256:d1f481f3a6e9868dd3ad65da265238ee109ab3a5dd7508958d5811953696f452"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Linux_arm64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915757"
 
 [tools."aqua:vektra/mockery"."platforms.linux-arm64-musl"]
-checksum = "sha256:ec02ca72b7ad1ff5a1de5503a27d3b1b3f230d722eb35c5b92e43e295a8e1290"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Linux_arm64.tar.gz"
+checksum = "sha256:d1f481f3a6e9868dd3ad65da265238ee109ab3a5dd7508958d5811953696f452"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Linux_arm64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915757"
 
 [tools."aqua:vektra/mockery"."platforms.linux-x64"]
-checksum = "sha256:ebce416b0175338525246c376885a1579ca6cd4d4015140ba0c70e6b5339a39c"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Linux_x86_64.tar.gz"
+checksum = "sha256:953c7d38b58dd6398b1c595f27caf8bbf27e9b733b49c404d2e124e86d2ae72a"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Linux_x86_64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915756"
 
 [tools."aqua:vektra/mockery"."platforms.linux-x64-musl"]
-checksum = "sha256:ebce416b0175338525246c376885a1579ca6cd4d4015140ba0c70e6b5339a39c"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Linux_x86_64.tar.gz"
+checksum = "sha256:953c7d38b58dd6398b1c595f27caf8bbf27e9b733b49c404d2e124e86d2ae72a"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Linux_x86_64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915756"
 
 [tools."aqua:vektra/mockery"."platforms.macos-arm64"]
-checksum = "sha256:a3a94b14c7414e148f2252199ffc4a0108d311358f3d336cbe05bb73cb203704"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Darwin_arm64.tar.gz"
+checksum = "sha256:303a794484243906b37da0c37056f974bf94278c2e10d7b1372e6bd53a2d5034"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Darwin_arm64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915806"
 
 [tools."aqua:vektra/mockery"."platforms.macos-x64"]
-checksum = "sha256:8bd2b3e4722e476400395920fb5bbd68be4ea22ec12cddf1e424c40610fda863"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Darwin_x86_64.tar.gz"
+checksum = "sha256:6882331344e946268b304dce042b01371a0213a896536413be8abe49d532ca6f"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Darwin_x86_64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915746"
 
 [tools."aqua:vektra/mockery"."platforms.windows-x64"]
-checksum = "sha256:3b4264066db615a56a8b59abda992f6db82b8cd7666937ed9dd8f4a4bfc424a0"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Windows_x86_64.tar.gz"
+checksum = "sha256:7a7458756e9d7bce5f9936b7a63394000f3affa751b9740e65350c4bdcb627dc"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Windows_x86_64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915759"
 
 [[tools.go]]
 version = "1.26.1"
@@ -71,32 +78,44 @@ backend = "aqua:golangci/golangci-lint"
 [tools.golangci-lint."platforms.linux-arm64"]
 checksum = "sha256:c3c83c7ed003e881991af21361c487151abd37969927ed9062cd0b48f3cd1d8d"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-linux-arm64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099658"
+provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.linux-arm64-musl"]
 checksum = "sha256:c3c83c7ed003e881991af21361c487151abd37969927ed9062cd0b48f3cd1d8d"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-linux-arm64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099658"
+provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.linux-x64"]
 checksum = "sha256:852741ea769f4543ab4b6cf86b313cb7173f2e8d2d39f00eb0cd5ef5d4d5eda9"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-linux-amd64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099674"
 provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.linux-x64-musl"]
 checksum = "sha256:852741ea769f4543ab4b6cf86b313cb7173f2e8d2d39f00eb0cd5ef5d4d5eda9"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-linux-amd64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099674"
+provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.macos-arm64"]
 checksum = "sha256:11e9549b955bae4bfb71aa3b5c2762e446e2d3f975fe963d6d767b8f745d9344"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-darwin-arm64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099671"
 provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.macos-x64"]
 checksum = "sha256:1dbbbe745eadba9cf012d49bd0b8c9290cbcb50dbf2cf9611f75a715b46ae3af"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-darwin-amd64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099653"
+provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.windows-x64"]
 checksum = "sha256:60e50178253d276d1c808ef00845eddcabca63b7d69d86a6dbaa41e88bc843f3"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-windows-amd64.zip"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099651"
+provenance = "github-attestations"
 
 [[tools.goreleaser]]
 version = "2.15.2"
@@ -105,37 +124,44 @@ backend = "aqua:goreleaser/goreleaser"
 [tools.goreleaser."platforms.linux-arm64"]
 checksum = "sha256:5db66761a98f6693161e49e1a95d28d2673a892ba60cb4a5e16736cafd41c4c9"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Linux_arm64.tar.gz"
-provenance = "cosign"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702105"
+provenance = "github-attestations"
 
 [tools.goreleaser."platforms.linux-arm64-musl"]
 checksum = "sha256:5db66761a98f6693161e49e1a95d28d2673a892ba60cb4a5e16736cafd41c4c9"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Linux_arm64.tar.gz"
-provenance = "cosign"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702105"
+provenance = "github-attestations"
 
 [tools.goreleaser."platforms.linux-x64"]
 checksum = "sha256:0ebdbf0353aba566b969dde746cc4e4806f96c27aa2f3971b229a9df7611fedc"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Linux_x86_64.tar.gz"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702106"
 provenance = "github-attestations"
 
 [tools.goreleaser."platforms.linux-x64-musl"]
 checksum = "sha256:0ebdbf0353aba566b969dde746cc4e4806f96c27aa2f3971b229a9df7611fedc"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Linux_x86_64.tar.gz"
-provenance = "cosign"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702106"
+provenance = "github-attestations"
 
 [tools.goreleaser."platforms.macos-arm64"]
 checksum = "sha256:0e6bd67688ac949780bf1166813a91f89856898ef4c40d7d46c2c74ebaa4b9ee"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Darwin_all.tar.gz"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702148"
 provenance = "github-attestations"
 
 [tools.goreleaser."platforms.macos-x64"]
 checksum = "sha256:0e6bd67688ac949780bf1166813a91f89856898ef4c40d7d46c2c74ebaa4b9ee"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Darwin_all.tar.gz"
-provenance = "cosign"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702148"
+provenance = "github-attestations"
 
 [tools.goreleaser."platforms.windows-x64"]
 checksum = "sha256:7459832946dbe122c144f8d7f87484d8572ca005b779310aa6bb03346e8de17a"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Windows_x86_64.zip"
-provenance = "cosign"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702135"
+provenance = "github-attestations"
 
 [[tools.node]]
 version = "22.20.0"
@@ -146,16 +172,17 @@ checksum = "sha256:4181609e03dcb9880e7e5bf956061ecc0503c77a480c6631d868cb1f65a2c
 url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0-linux-arm64.tar.gz"
 
 [tools.node."platforms.linux-arm64-musl"]
-checksum = "sha256:4181609e03dcb9880e7e5bf956061ecc0503c77a480c6631d868cb1f65a2c7dd"
-url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0-linux-arm64.tar.gz"
+checksum = "sha256:79a087c1e23fc97d173ad6676df5b1a4e1a02daf1acca05ee25662747c5d492c"
+url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0.tar.gz"
+install = "source"
 
 [tools.node."platforms.linux-x64"]
 checksum = "sha256:eeaccb0378b79406f2208e8b37a62479c70595e20be6b659125eb77dd1ab2a29"
 url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0-linux-x64.tar.gz"
 
 [tools.node."platforms.linux-x64-musl"]
-checksum = "sha256:eeaccb0378b79406f2208e8b37a62479c70595e20be6b659125eb77dd1ab2a29"
-url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0-linux-x64.tar.gz"
+checksum = "sha256:461aafed5eb607357479c6de5df978515d74daf7e0169e972e61c8390c035f1a"
+url = "https://unofficial-builds.nodejs.org/download/release/v22.20.0/node-v22.20.0-linux-x64-musl.tar.gz"
 
 [tools.node."platforms.macos-arm64"]
 checksum = "sha256:cc04a76a09f79290194c0646f48fec40354d88969bec467789a5d55dd097f949"
@@ -173,9 +200,34 @@ url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0-win-x64.zip"
 version = "27.3"
 backend = "aqua:protocolbuffers/protobuf/protoc"
 
+[tools.protoc."platforms.linux-arm64"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-linux-aarch_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006374"
+
+[tools.protoc."platforms.linux-arm64-musl"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-linux-aarch_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006374"
+
 [tools.protoc."platforms.linux-x64"]
 checksum = "blake3:8c2938accf7061164ea3968529a47ced1db4824ca553857f6903c68ae4347739"
 url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-linux-x86_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006380"
+
+[tools.protoc."platforms.linux-x64-musl"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-linux-x86_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006380"
+
+[tools.protoc."platforms.macos-arm64"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-osx-aarch_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006383"
+
+[tools.protoc."platforms.macos-x64"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-osx-x86_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006381"
+
+[tools.protoc."platforms.windows-x64"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-win64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006385"
 
 [[tools.protoc-gen-go]]
 version = "1.34.2"
@@ -183,30 +235,62 @@ backend = "aqua:protocolbuffers/protobuf-go/protoc-gen-go"
 
 [tools.protoc-gen-go."platforms.linux-arm64"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.linux.arm64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109690"
 
 [tools.protoc-gen-go."platforms.linux-arm64-musl"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.linux.arm64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109690"
 
 [tools.protoc-gen-go."platforms.linux-x64"]
 checksum = "blake3:f765e393e0f0b25070c13848d88f5e6303e66090132afa1b753e981507a4174e"
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.linux.amd64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109692"
 
 [tools.protoc-gen-go."platforms.linux-x64-musl"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.linux.amd64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109692"
 
 [tools.protoc-gen-go."platforms.macos-arm64"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.darwin.arm64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109697"
 
 [tools.protoc-gen-go."platforms.macos-x64"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.darwin.amd64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109698"
 
 [tools.protoc-gen-go."platforms.windows-x64"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.windows.amd64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109685"
 
 [[tools.protoc-gen-go-grpc]]
 version = "1.5.1"
 backend = "aqua:grpc/grpc-go/protoc-gen-go-grpc"
 
+[tools.protoc-gen-go-grpc."platforms.linux-arm64"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.linux.arm64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572238"
+
+[tools.protoc-gen-go-grpc."platforms.linux-arm64-musl"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.linux.arm64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572238"
+
 [tools.protoc-gen-go-grpc."platforms.linux-x64"]
 checksum = "blake3:197739132f0408f017d0c83773ac1ace85c0572473fdd658ec6e4ccbf20e0552"
 url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.linux.amd64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572229"
+
+[tools.protoc-gen-go-grpc."platforms.linux-x64-musl"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.linux.amd64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572229"
+
+[tools.protoc-gen-go-grpc."platforms.macos-arm64"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.darwin.arm64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572243"
+
+[tools.protoc-gen-go-grpc."platforms.macos-x64"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.darwin.amd64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572208"
+
+[tools.protoc-gen-go-grpc."platforms.windows-x64"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.windows.amd64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572216"

mise.toml

@@ -2,7 +2,7 @@
 experimental = true
 
 [tools]
-"aqua:vektra/mockery" = "3.5.3"
+"aqua:vektra/mockery" = "3.8.1"
 go = "1.26.1"
 golangci-lint = "2.11.2"
 goreleaser = "2.15.2"
@@ -40,6 +40,10 @@ outputs = ["uncloudd-linux-arm64"]
 description = "Update mise.lock with hashes for all platforms"
 run = "mise lock --platform linux-x64,macos-arm64"
 
+[tasks.mocks]
+description = "Regenerate Go test mocks"
+run = "mockery"
+
 [tasks.proto]
 description = "Regenerate gRPC API code from .proto files"
 run = """

pkg/api/client.go

@@ -42,6 +42,8 @@ type DNSClient interface {
 type ImageClient interface {
 	InspectImage(ctx context.Context, id string) ([]MachineImage, error)
 	InspectRemoteImage(ctx context.Context, id string) ([]MachineRemoteImage, error)
+	ListImages(ctx context.Context, filter ImageFilter) ([]MachineImages, error)
+	RemoveImage(ctx context.Context, image string, opts RemoveImageOptions) ([]MachineImageRemoval, error)
 }
 
 type LogsClient interface {

pkg/api/image.go

@@ -32,26 +32,16 @@ type ImageFilter struct {
 	Name string
 }
 
-// ImageReferences returns the image tag and named digest references, sorted and without duplicates.
-// References use familiar names such as "alpine:latest" and "alpine@sha256:...".
-// Bare digests and invalid references are excluded. It returns nil if there are no named references.
-func ImageReferences(img image.Summary) []string {
-	var refs []string
-	for _, names := range [][]string{img.RepoTags, img.RepoDigests} {
-		for _, name := range names {
-			ref, err := reference.ParseAnyReference(name)
-			if err != nil {
-				continue
-			}
-			// RepoDigests can contain bare digests. Keep only references with a repository name.
-			if named, ok := ref.(reference.Named); ok {
-				refs = append(refs, reference.FamiliarString(named))
-			}
-		}
-	}
+// RemoveImageOptions defines options for removing an image from cluster machines.
+type RemoveImageOptions struct {
+	// Machines selects machines to remove the image from (names or IDs). If empty, all cluster machines are selected.
+	Machines []string
+}
 
-	slices.Sort(refs)
-	return slices.Compact(refs)
+// MachineImageRemoval contains the outcome of removing an image on one machine.
+type MachineImageRemoval struct {
+	Metadata *pb.Metadata
+	Results  []image.DeleteResponse
 }
 
 // MachineRemoteImage represents an image in a remote registry fetched by a particular machine.

pkg/client/image.go

@@ -23,9 +23,11 @@ import (
 	"github.com/docker/docker/api/types/image"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/jsonmessage"
+	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/go-connections/nat"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/psviderski/uncloud/api/pb"
+	cliprogress "github.com/psviderski/uncloud/internal/cli/progress"
 	"github.com/psviderski/uncloud/internal/cli/tui"
 	"github.com/psviderski/uncloud/internal/docker"
 	"github.com/psviderski/uncloud/internal/machine/constants"
@@ -34,6 +36,8 @@ import (
 	"github.com/psviderski/uncloud/internal/secret"
 	"github.com/psviderski/uncloud/pkg/api"
 	netproxy "golang.org/x/net/proxy"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 // This is the container image used to run socat proxy containers.
@@ -104,6 +108,80 @@ func (cli *Client) ListImages(ctx context.Context, filter api.ImageFilter) ([]ap
 	return machineImages, nil
 }
 
+// RemoveImage removes one exact image name or ID on the selected machines with protection of used references.
+// Results include the outcome of removing an image on each machine.
+//
+// An absent image on some machines is normal if removal succeeds on at least one machine. If every selected machine
+// reports NotFound, the returned error is api.ErrNotFound. Other machine failures are combined, with successful
+// results still returned.
+func (cli *Client) RemoveImage(
+	ctx context.Context, imageNameOrID string, opts api.RemoveImageOptions,
+) ([]api.MachineImageRemoval, error) {
+	if strings.TrimSpace(imageNameOrID) == "" {
+		return nil, errors.New("image name or ID is required")
+	}
+
+	resp, err := cli.Docker.GRPCClient.RemoveImage(ProxyMachinesContext(ctx, opts.Machines),
+		&pb.RemoveImageRequest{Id: imageNameOrID, PruneChildren: true})
+	if err != nil {
+		return nil, err
+	}
+
+	pw := progress.ContextWriter(ctx)
+	var results []api.MachineImageRemoval
+	var errs []error
+	notFoundCount := 0
+	for _, msg := range resp.Messages {
+		result := api.MachineImageRemoval{Metadata: msg.Metadata}
+		for _, removal := range msg.Results {
+			result.Results = append(result.Results, image.DeleteResponse{
+				Deleted:  removal.Deleted,
+				Untagged: removal.Untagged,
+			})
+		}
+		results = append(results, result)
+
+		eventID := cliprogress.ImageEventID(imageNameOrID, msg.Metadata.MachineName)
+		if err = result.Metadata.Err(); err != nil {
+			event := progress.NewEvent(eventID, progress.Done, "Not found")
+			if status.Code(err) == codes.NotFound {
+				notFoundCount++
+			} else {
+				errs = append(errs, fmt.Errorf("machine '%s': %w", msg.Metadata.MachineName, err))
+				event.Status = progress.Error
+				event.StatusText = status.Convert(err).Message()
+			}
+			pw.Event(event)
+			continue
+		}
+
+		// An unnamed image can return Deleted without Untagged. Use the requested name or ID as its row fallback.
+		deleted := false
+		for _, removal := range result.Results {
+			if removal.Untagged != "" {
+				eventID = cliprogress.ImageEventID(removal.Untagged, msg.Metadata.MachineName)
+				pw.Event(progress.NewEvent(eventID, progress.Done, "Untagged"))
+			}
+			if removal.Deleted != "" && !deleted {
+				// Docker reports the deletion of the selected image before any pruned parents or released layers.
+				// Only that first deletion updates the last untagged row marking the image as Removed.
+				deleted = true
+				pw.Event(progress.NewEvent(eventID, progress.Done,
+					fmt.Sprintf("Removed (%s)", stringid.TruncateID(removal.Deleted))))
+			}
+		}
+	}
+
+	if err = errors.Join(errs...); err != nil {
+		return results, err
+	}
+	if notFoundCount == len(results) {
+		return results, api.ErrNotFound
+	}
+
+	return results, nil
+}
+
 // TODO: move PushImageOptions to api package and include PushImage in ImageClient interface.
 type PushImageOptions struct {
 	// AllMachines pushes the image to all machines in the cluster. Takes precedence over Machines field.

test/e2e/service_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/go-units"
@@ -1265,10 +1266,10 @@ myapp.example.com {
 			"Containers should be on machines where the volume exists")
 	})
 
-	t.Run("list images for replicated service", func(t *testing.T) {
+	t.Run("image lifecycle for replicated service", func(t *testing.T) {
 		t.Parallel()
 
-		serviceName := "test-list-images-replicated"
+		serviceName := "test-image-lifecycle-replicated"
 		uniqueImage := "portainer/pause:3.9" // Unique image not used by other tests.
 		t.Cleanup(func() {
 			err := cli.RemoveService(ctx, serviceName)
@@ -1301,39 +1302,59 @@ myapp.example.com {
 		machinesWithContainers := serviceMachines(svc)
 		assert.Len(t, machinesWithContainers.ToSlice(), 2, "Containers should be on 2 different machines")
 
-		var machineWithoutContainer string
-		for _, m := range c.Machines {
-			if !machinesWithContainers.Contains(m.ID) {
-				machineWithoutContainer = m.ID
-				break
-			}
-		}
-		assert.NotEmpty(t, machineWithoutContainer, "Should have found a machine without container")
-
-		machineImages, err := cli.ListImages(ctx, api.ImageFilter{})
+		machineImages, err := cli.ListImages(ctx, api.ImageFilter{Name: uniqueImage})
 		require.NoError(t, err)
-		assert.Len(t, machineImages, 3, "Should get images from all 3 machines")
+		require.Len(t, machineImages, len(c.Machines))
 
 		for _, mi := range machineImages {
-			// Checking only DockerImages because the machines in ucind cluster don't use the containerd image store.
+			require.NoError(t, mi.Metadata.Err())
 			if !machinesWithContainers.Contains(mi.Metadata.MachineId) {
-				// This is the machine without service containers, it should not have the unique image.
-				for _, img := range mi.Images {
-					assert.NotContains(t, img.RepoTags, uniqueImage)
-				}
+				assert.Empty(t, mi.Images)
 				continue
 			}
 
-			// Check if the unique image is present on the machine where a service container is running.
-			hasImage := false
-			for _, img := range mi.Images {
-				if slices.Contains(img.RepoTags, uniqueImage) {
-					hasImage = true
-					break
+			require.Len(t, mi.Images, 1)
+			assert.Contains(t, mi.Images[0].RepoTags, uniqueImage)
 		}
+
+		// The used tag must remain on both machines while their service containers exist.
+		results, err := cli.RemoveImage(ctx, uniqueImage, api.RemoveImageOptions{Machines: spec.Placement.Machines})
+		require.Error(t, err)
+		assert.ErrorContains(t, err, "used by container")
+		require.Len(t, results, 2)
+		for _, result := range results {
+			assert.Equal(t, codes.FailedPrecondition, status.Code(result.Metadata.Err()))
 		}
-			assert.True(t, hasImage, "Machine %s with container should have image %s",
-				mi.Metadata.MachineId, uniqueImage)
+
+		require.NoError(t, cli.RemoveService(ctx, serviceName))
+
+		// Broadcast removal after deleting the containers. Absence on the third machine is not an overall error.
+		results, err = cli.RemoveImage(ctx, uniqueImage, api.RemoveImageOptions{})
+		require.NoError(t, err)
+		require.Len(t, results, len(c.Machines))
+		for _, result := range results {
+			if !machinesWithContainers.Contains(result.Metadata.MachineId) {
+				assert.Equal(t, codes.NotFound, status.Code(result.Metadata.Err()))
+				continue
+			}
+			require.NoError(t, result.Metadata.Err())
+			assert.Contains(t, result.Results, image.DeleteResponse{Untagged: uniqueImage})
+		}
+
+		results, err = cli.RemoveImage(ctx, uniqueImage, api.RemoveImageOptions{})
+		require.ErrorIs(t, err, api.ErrNotFound)
+		require.Len(t, results, len(c.Machines))
+		for _, result := range results {
+			assert.Equal(t, codes.NotFound, status.Code(result.Metadata.Err()))
+			assert.Empty(t, result.Results)
+		}
+
+		machineImages, err = cli.ListImages(ctx, api.ImageFilter{Name: uniqueImage})
+		require.NoError(t, err)
+		require.Len(t, machineImages, len(c.Machines))
+		for _, mi := range machineImages {
+			require.NoError(t, mi.Metadata.Err())
+			assert.Empty(t, mi.Images)
 		}
 	})