feat(images): implement RemoveImage gRPC API with extra protection for used images

Signed-off-by: Miek Gieben <miek@miek.nl>

.mockery.yml

@@ -1,4 +1,10 @@
 packages:
+  github.com/docker/docker/client:
+    config:
+      dir: internal/machine/docker
+      pkgname: docker
+    interfaces:
+      APIClient:
   github.com/psviderski/uncloud/internal/machine/caddyconfig:
     interfaces:
       CaddyfileValidator:

HACKING.md

@@ -116,7 +116,7 @@ mise run proto
 Regenerate mocks:
 
 ```shell
-make mocks
+mise run mocks
 ```
 
 ## Uncloud in Docker (UCinD)

Makefile

@@ -21,10 +21,6 @@ corrosion-multiarch-image-push:
 ucind-multiarch-image-push:
 	docker buildx build --push --platform linux/amd64,linux/arm64 -t "$(UCIND_IMAGE)" --target ucind .
 
-.PHONY: mocks
-mocks:
-	@mockery
-
 .PHONY: test
 test:
 ifeq ($(TEST_NAME),)

api/pb/common.go

@@ -4,8 +4,16 @@ import (
 	"bytes"
 	"fmt"
 	"net/netip"
+
+	"google.golang.org/grpc/status"
 )
 
+// Err returns the upstream error, preserving its gRPC status code.
+// It returns nil for nil metadata or an absent or successful status.
+func (m *Metadata) Err() error {
+	return status.ErrorProto(m.GetStatus())
+}
+
 func NewIP(addr netip.Addr) *IP {
 	// MarshalBinary always returns a nil error.
 	ip, _ := addr.MarshalBinary()

api/pb/common.pb.go

@@ -87,7 +87,8 @@ type Metadata struct {
 	MachineName string `protobuf:"bytes,5,opt,name=machine_name,json=machineName,proto3" json:"machine_name,omitempty"`
 	// Address of the machine the response came from.
 	MachineAddr string `protobuf:"bytes,1,opt,name=machine_addr,json=machineAddr,proto3" json:"machine_addr,omitempty"`
-	// error is set if the request to upstream failed. The rest of the response is undefined.
+	// error is set if the request to upstream failed. The rest of the response apart from this metadata message
+	// is undefined.
 	Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
 	// error as a gRPC Status message.
 	Status *status.Status `protobuf:"bytes,3,opt,name=status,proto3" json:"status,omitempty"`

api/pb/common.proto

@@ -18,7 +18,8 @@ message Metadata {
   // Address of the machine the response came from.
   string machine_addr = 1;
 
-  // error is set if the request to upstream failed. The rest of the response is undefined.
+  // error is set if the request to upstream failed. The rest of the response apart from this metadata message
+  // is undefined.
   string error = 2;
   // error as a gRPC Status message.
   google.rpc.Status status = 3;

api/pb/docker.proto

@@ -24,6 +24,7 @@ service Docker {
   // Docker auth credentials if necessary.
   rpc InspectRemoteImage(InspectRemoteImageRequest) returns (InspectRemoteImageResponse);
   rpc ListImages(ListImagesRequest) returns (ListImagesResponse);
+  rpc RemoveImage(RemoveImageRequest) returns (RemoveImageResponse);
 
   rpc CreateVolume(CreateVolumeRequest) returns (CreateVolumeResponse);
   rpc ListVolumes(ListVolumesRequest) returns (ListVolumesResponse);
@@ -179,6 +180,40 @@ message ListImagesRequest {
   bytes options = 1;
 }
 
+message RemoveImageRequest {
+  string id = 1;
+  bool force = 2;
+  bool prune_children = 3;
+  repeated ImagePlatform platforms = 4;
+}
+
+// ImagePlatform describes an OCI image platform.
+message ImagePlatform {
+  string architecture = 1;
+  string os = 2;
+  string os_version = 3;
+  repeated string os_features = 4;
+  string variant = 5;
+}
+
+message RemoveImageResponse {
+  // Must contain only one repeated messages field to allow broadcasting RemoveImage requests to multiple machines.
+  repeated MachineImageRemoval messages = 1;
+}
+
+message MachineImageRemoval {
+  Metadata metadata = 1;
+  repeated ImageRemoval results = 2;
+}
+
+// ImageRemoval reports an image deletion or an untagged image reference.
+message ImageRemoval {
+  // The image ID of an image that was deleted.
+  string deleted = 1;
+  // The image reference of an image that was untagged.
+  string untagged = 2;
+}
+
 message ListImagesResponse {
   // Must contain only one repeated messages field to allow broadcasting ListImages requests to multiple machines.
   repeated MachineImages messages = 1;

api/pb/docker_grpc.pb.go

@@ -32,6 +32,7 @@ const (
 	Docker_InspectImage_FullMethodName            = "/api.Docker/InspectImage"
 	Docker_InspectRemoteImage_FullMethodName      = "/api.Docker/InspectRemoteImage"
 	Docker_ListImages_FullMethodName              = "/api.Docker/ListImages"
+	Docker_RemoveImage_FullMethodName             = "/api.Docker/RemoveImage"
 	Docker_CreateVolume_FullMethodName            = "/api.Docker/CreateVolume"
 	Docker_ListVolumes_FullMethodName             = "/api.Docker/ListVolumes"
 	Docker_RemoveVolume_FullMethodName            = "/api.Docker/RemoveVolume"
@@ -59,6 +60,7 @@ type DockerClient interface {
 	// Docker auth credentials if necessary.
 	InspectRemoteImage(ctx context.Context, in *InspectRemoteImageRequest, opts ...grpc.CallOption) (*InspectRemoteImageResponse, error)
 	ListImages(ctx context.Context, in *ListImagesRequest, opts ...grpc.CallOption) (*ListImagesResponse, error)
+	RemoveImage(ctx context.Context, in *RemoveImageRequest, opts ...grpc.CallOption) (*RemoveImageResponse, error)
 	CreateVolume(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error)
 	ListVolumes(ctx context.Context, in *ListVolumesRequest, opts ...grpc.CallOption) (*ListVolumesResponse, error)
 	RemoveVolume(ctx context.Context, in *RemoveVolumeRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
@@ -217,6 +219,16 @@ func (c *dockerClient) ListImages(ctx context.Context, in *ListImagesRequest, op
 	return out, nil
 }
 
+func (c *dockerClient) RemoveImage(ctx context.Context, in *RemoveImageRequest, opts ...grpc.CallOption) (*RemoveImageResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(RemoveImageResponse)
+	err := c.cc.Invoke(ctx, Docker_RemoveImage_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *dockerClient) CreateVolume(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error) {
 	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
 	out := new(CreateVolumeResponse)
@@ -305,6 +317,7 @@ type DockerServer interface {
 	// Docker auth credentials if necessary.
 	InspectRemoteImage(context.Context, *InspectRemoteImageRequest) (*InspectRemoteImageResponse, error)
 	ListImages(context.Context, *ListImagesRequest) (*ListImagesResponse, error)
+	RemoveImage(context.Context, *RemoveImageRequest) (*RemoveImageResponse, error)
 	CreateVolume(context.Context, *CreateVolumeRequest) (*CreateVolumeResponse, error)
 	ListVolumes(context.Context, *ListVolumesRequest) (*ListVolumesResponse, error)
 	RemoveVolume(context.Context, *RemoveVolumeRequest) (*emptypb.Empty, error)
@@ -358,6 +371,9 @@ func (UnimplementedDockerServer) InspectRemoteImage(context.Context, *InspectRem
 func (UnimplementedDockerServer) ListImages(context.Context, *ListImagesRequest) (*ListImagesResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method ListImages not implemented")
 }
+func (UnimplementedDockerServer) RemoveImage(context.Context, *RemoveImageRequest) (*RemoveImageResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method RemoveImage not implemented")
+}
 func (UnimplementedDockerServer) CreateVolume(context.Context, *CreateVolumeRequest) (*CreateVolumeResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method CreateVolume not implemented")
 }
@@ -591,6 +607,24 @@ func _Docker_ListImages_Handler(srv interface{}, ctx context.Context, dec func(i
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Docker_RemoveImage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(RemoveImageRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(DockerServer).RemoveImage(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: Docker_RemoveImage_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(DockerServer).RemoveImage(ctx, req.(*RemoveImageRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _Docker_CreateVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(CreateVolumeRequest)
 	if err := dec(in); err != nil {
@@ -760,6 +794,10 @@ var Docker_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "ListImages",
 			Handler:    _Docker_ListImages_Handler,
 		},
+		{
+			MethodName: "RemoveImage",
+			Handler:    _Docker_RemoveImage_Handler,
+		},
 		{
 			MethodName: "CreateVolume",
 			Handler:    _Docker_CreateVolume_Handler,

cmd/uc/caddy/config.go

@@ -3,8 +3,8 @@ package caddy
 import (
 	"context"
 	"fmt"
-	"os"
 
+	"charm.land/lipgloss/v2"
 	"github.com/alecthomas/chroma/v2/quick"
 	"github.com/psviderski/uncloud/internal/cli"
 	"github.com/psviderski/uncloud/internal/cli/completion"
@@ -15,7 +15,6 @@ import (
 
 type configOptions struct {
 	machine string
-	noColor bool
 }
 
 func NewConfigCommand() *cobra.Command {
@@ -33,8 +32,6 @@ func NewConfigCommand() *cobra.Command {
 
 	cmd.Flags().StringVarP(&opts.machine, "machine", "m", "",
 		"Name or ID of the machine to get the configuration from. (default is connected machine)")
-	cmd.Flags().BoolVar(&opts.noColor, "no-color", false,
-		"Disable syntax highlighting for the output.")
 
 	completion.MachinesFlag(cmd)
 
@@ -54,13 +51,9 @@ func runConfig(ctx context.Context, uncli *cli.CLI, opts configOptions) error {
 	}
 
 	// Print the Caddyfile with syntax highlighting.
-	if opts.noColor {
-		fmt.Print(config.Caddyfile)
-	} else {
-		if err = quick.Highlight(os.Stdout, config.Caddyfile, "caddy", "terminal256", "monokai"); err != nil {
-			// If highlighting fails, fall back to plain output.
-			fmt.Print(config.Caddyfile)
-		}
+	if err = quick.Highlight(lipgloss.Writer, config.Caddyfile, "caddy", "terminal256", "monokai"); err != nil {
+		// If highlighting fails, fall back to plain output.
+		lipgloss.Print(config.Caddyfile)
 	}
 
 	if config.LastReconciliationError != "" {

cmd/uc/context/show.go

@@ -22,15 +22,6 @@ func NewShowCommand() *cobra.Command {
 }
 
 func show(uncli *cli.CLI) error {
-	// discard errors, only show the current context, otherwise nothing
-	if uncli.Config == nil {
-		return nil
-	}
-
-	if len(uncli.Config.Contexts) == 0 {
-		return nil
-	}
-	fmt.Println(uncli.Config.CurrentContext)
-
+	fmt.Println(uncli.ContextOverrideOrCurrent())
 	return nil
 }

cmd/uc/deploy.go

@@ -18,7 +18,7 @@ import (
 	"github.com/psviderski/uncloud/pkg/client"
 	"github.com/psviderski/uncloud/pkg/client/compose"
 	"github.com/psviderski/uncloud/pkg/client/deploy"
-	"github.com/psviderski/uncloud/pkg/client/deploy/operation"
+	"github.com/psviderski/uncloud/pkg/client/operation"
 	"github.com/spf13/cobra"
 )
 
@@ -305,7 +305,7 @@ const defaultFailedContainerLogsTail = 10
 // UNCLOUD_FAILED_CONTAINER_LOGS_TAIL environment variable override when set and valid.
 func failedContainerLogsTail() int {
 	if v := os.Getenv("UNCLOUD_FAILED_CONTAINER_LOGS_TAIL"); v != "" {
-		if tail, err := logs.Tail(v); err == nil && (tail == -1 || tail > 0) {
+		if tail, err := logs.ParseTail(v); err == nil && (tail == -1 || tail > 0) {
 			return tail
 		}
 	}

cmd/uc/image/ls.go

@@ -13,6 +13,7 @@ import (
 	"github.com/charmbracelet/colorprofile"
 	"github.com/containerd/platforms"
 	"github.com/docker/docker/api/types/image"
+	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/go-units"
 
 	"github.com/psviderski/uncloud/internal/cli"
@@ -102,7 +103,7 @@ func list(ctx context.Context, uncli *cli.CLI, opts listOptions) error {
 	var rows []imageRow
 
 	for _, machineImages := range clusterImages {
-		if err := machineImages.Error(); err != nil {
+		if err := machineImages.Metadata.Err(); err != nil {
 			tui.PrintWarning(fmt.Sprintf("failed to list images on machine '%s': %s",
 				machineImages.Metadata.MachineName, err))
 			continue
@@ -116,49 +117,7 @@ func list(ctx context.Context, uncli *cli.CLI, opts listOptions) error {
 			store = "containerd"
 		}
 
-		// Process each image for this machine.
-		for _, img := range machineImages.Images {
-			// Show the first 12 chars without 'sha256:' as the image ID like Docker does.
-			id := strings.TrimPrefix(img.ID, "sha256:")[:12]
-
-			name := "<none>"
-			if len(img.RepoTags) > 0 && img.RepoTags[0] != "<none>:<none>" {
-				name = img.RepoTags[0]
-			}
-
-			imgPlatforms, _ := imagePlatforms(img)
-			formattedPlatforms := formatPlatforms(imgPlatforms)
-
-			created := ""
-			createdAt := time.Unix(img.Created, 0)
-			if !createdAt.IsZero() {
-				created = units.HumanDuration(time.Now().UTC().Sub(createdAt)) + " ago"
-			}
-
-			size := units.HumanSizeWithPrecision(float64(img.Size), 3)
-
-			// Check if the image is in use by any containers. Only supported by Docker API >=1.51
-			inUse := "-"
-			if img.Containers != -1 { // -1 means the info is not available.
-				if img.Containers > 0 {
-					inUse = lipgloss.NewStyle().Foreground(lipgloss.Color("10")).Render("●")
-				} else {
-					inUse = lipgloss.NewStyle().Foreground(lipgloss.Color("8")).Render("○")
-				}
-			}
-
-			rows = append(rows, imageRow{
-				id:           id,
-				name:         name,
-				platforms:    formattedPlatforms,
-				createdHuman: created,
-				createdUnix:  img.Created,
-				size:         size,
-				inUse:        inUse,
-				store:        store,
-				machine:      machineName,
-			})
-		}
+		rows = append(rows, imageRows(machineImages.Images, machineName, store)...)
 	}
 
 	if len(rows) == 0 {
@@ -184,13 +143,55 @@ func list(ctx context.Context, uncli *cli.CLI, opts listOptions) error {
 	return nil
 }
 
+// imageRows keeps every stored name visible and retains digest-only and unnamed images.
+func imageRows(images []image.Summary, machineName, store string) []imageRow {
+	var rows []imageRow
+	for _, img := range images {
+		// RepoTags includes stored digest names on containerd. RepoDigests can include derived names.
+		imageNames := img.RepoTags
+		if len(imageNames) == 0 {
+			imageNames = []string{"<untagged>"}
+		}
+
+		imgPlatforms, _ := imagePlatforms(img)
+		created := ""
+		if img.Created > 0 {
+			created = units.HumanDuration(time.Since(time.Unix(img.Created, 0))) + " ago"
+		}
+		// Check if the image is in use by any containers. Only supported by Docker API >=1.51
+		inUse := "-"
+		if img.Containers >= 0 { // -1 means the info is not available.
+			inUse = lipgloss.NewStyle().Foreground(lipgloss.Color("8")).Render("○")
+			if img.Containers > 0 {
+				inUse = lipgloss.NewStyle().Foreground(lipgloss.Color("10")).Render("●")
+			}
+		}
+
+		for _, name := range imageNames {
+			rows = append(rows, imageRow{
+				id:           stringid.TruncateID(img.ID),
+				name:         name,
+				platforms:    formatPlatforms(imgPlatforms),
+				createdHuman: created,
+				createdUnix:  img.Created,
+				size:         units.HumanSizeWithPrecision(float64(img.Size), 3),
+				inUse:        inUse,
+				store:        store,
+				machine:      machineName,
+			})
+		}
+	}
+
+	return rows
+}
+
 // imagePlatforms returns a list of platforms supported by the image and a boolean indicating if it's multi-platform.
 func imagePlatforms(img image.Summary) ([]string, bool) {
 	var formattedPlatforms []string
 	multiPlatform := false
 
 	for _, m := range img.Manifests {
-		if m.Kind != image.ManifestKindImage || !m.Available {
+		if m.Kind != image.ManifestKindImage || !m.Available || m.ImageData == nil {
 			continue
 		}
 

cmd/uc/machine/logs.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"slices"
 	"strings"
+	"time"
 
 	"github.com/psviderski/uncloud/internal/cli"
 	"github.com/psviderski/uncloud/internal/cli/completion"
@@ -64,6 +65,15 @@ If no services are specified, streams logs from the uncloud service.`,
 }
 
 func runLogs(ctx context.Context, uncli *cli.CLI, services []string, opts logs.Options) error {
+	since, until, err := logs.TimeRange(opts.Since, opts.Until, time.Now())
+	if err != nil {
+		return err
+	}
+	tail, err := opts.TailLines()
+	if err != nil {
+		return err
+	}
+
 	if len(services) == 0 {
 		services = []string{api.SystemServiceUncloud}
 	}
@@ -74,11 +84,6 @@ func runLogs(ctx context.Context, uncli *cli.CLI, services []string, opts logs.O
 		}
 	}
 
-	tail, err := logs.Tail(opts.Tail)
-	if err != nil {
-		return err
-	}
-
 	c, err := uncli.ConnectCluster(ctx)
 	if err != nil {
 		return fmt.Errorf("connect to cluster: %w", err)
@@ -88,8 +93,8 @@ func runLogs(ctx context.Context, uncli *cli.CLI, services []string, opts logs.O
 	logsOpts := api.ServiceLogsOptions{
 		Follow:   opts.Follow,
 		Tail:     tail,
-		Since:    opts.Since,
-		Until:    opts.Until,
+		Since:    since,
+		Until:    until,
 		Machines: cli.ExpandCommaSeparatedValues(opts.Machines),
 	}
 

cmd/uc/machine/logs_test.go

@@ -0,0 +1,25 @@
+package machine
+
+import (
+	"context"
+	"testing"
+
+	"github.com/psviderski/uncloud/internal/cli/logs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRunLogsInvalidTimeFilters(t *testing.T) {
+	t.Parallel()
+
+	// Invalid filters must fail before connecting to the cluster.
+	for _, flag := range []string{"since", "until"} {
+		opts := logs.Options{}
+		if flag == "since" {
+			opts.Since = "invalid"
+		} else {
+			opts.Until = "invalid"
+		}
+		err := runLogs(context.Background(), nil, nil, opts)
+		require.ErrorContains(t, err, "invalid --"+flag+" value")
+	}
+}

cmd/uc/machine/rtt.go

@@ -54,8 +54,8 @@ func rtt(ctx context.Context, uncli *cli.CLI) error {
 			tui.PrintWarning("metadata is missing in response from unknown server")
 			continue
 		}
-		if m.Metadata.Error != "" {
-			tui.PrintWarning(fmt.Sprintf("failed to inspect machine '%s': %s", m.Metadata.MachineName, m.Metadata.Error))
+		if err := m.Metadata.Err(); err != nil {
+			tui.PrintWarning(fmt.Sprintf("failed to inspect machine '%s': %s", m.Metadata.MachineName, err))
 			continue
 		}
 		if m.Machine == nil {

cmd/uc/ps.go

@@ -207,9 +207,9 @@ func collectContainers(ctx context.Context, cli *client.Client) ([]containerInfo
 
 		machineName := msc.Metadata.MachineName
 
-		if msc.Metadata.Error != "" {
+		if err := msc.Metadata.Err(); err != nil {
 			tui.PrintWarning(fmt.Sprintf("failed to list service containers on machine %s: %s",
-				machineName, msc.Metadata.Error))
+				machineName, err))
 			continue
 		}
 

cmd/uc/service/logs.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/psviderski/uncloud/internal/cli"
@@ -78,6 +79,15 @@ If no services are specified, streams logs from all services defined in the Comp
 }
 
 func RunLogs(ctx context.Context, uncli *cli.CLI, args []string, opts logs.Options) error {
+	since, until, err := logs.TimeRange(opts.Since, opts.Until, time.Now())
+	if err != nil {
+		return err
+	}
+	tail, err := opts.TailLines()
+	if err != nil {
+		return err
+	}
+
 	serviceArgs, err := logs.ParseServiceArgs(args)
 	if err != nil {
 		return err
@@ -106,12 +116,6 @@ func RunLogs(ctx context.Context, uncli *cli.CLI, args []string, opts logs.Optio
 		}
 	}
 
-	// Parse tail option.
-	tail, err := logs.Tail(opts.Tail)
-	if err != nil {
-		return err
-	}
-
 	c, err := uncli.ConnectCluster(ctx)
 	if err != nil {
 		return fmt.Errorf("connect to cluster: %w", err)
@@ -121,8 +125,8 @@ func RunLogs(ctx context.Context, uncli *cli.CLI, args []string, opts logs.Optio
 	baseOpts := api.ServiceLogsOptions{
 		Follow:   opts.Follow,
 		Tail:     tail,
-		Since:    opts.Since,
-		Until:    opts.Until,
+		Since:    since,
+		Until:    until,
 		Machines: cli.ExpandCommaSeparatedValues(opts.Machines),
 	}
 

cmd/uc/service/logs_test.go

@@ -0,0 +1,25 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"github.com/psviderski/uncloud/internal/cli/logs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRunLogsInvalidTimeFilters(t *testing.T) {
+	t.Parallel()
+
+	// Invalid filters must fail before loading Compose files or connecting to the cluster.
+	for _, flag := range []string{"since", "until"} {
+		opts := logs.Options{}
+		if flag == "since" {
+			opts.Since = "invalid"
+		} else {
+			opts.Until = "invalid"
+		}
+		err := RunLogs(context.Background(), nil, nil, opts)
+		require.ErrorContains(t, err, "invalid --"+flag+" value")
+	}
+}

internal/cli/logs/duration.go

@@ -0,0 +1,66 @@
+package logs
+
+import (
+	"fmt"
+	"math/big"
+	"strings"
+	"time"
+)
+
+// Duration is an extended standard time.Duration that also supports days as a unit.
+type Duration = time.Duration
+
+// ParseDuration parses a Go duration with the additional unit d, meaning exactly 24 hours.
+// Days can be fractional or combined with other units, such as "1.5d" or "2d3h".
+func ParseDuration(value string) (Duration, error) {
+	if !strings.Contains(value, "d") {
+		return time.ParseDuration(value)
+	}
+
+	rest := value
+	var normalised strings.Builder
+	if len(rest) > 0 && (rest[0] == '-' || rest[0] == '+') {
+		normalised.WriteByte(rest[0])
+		rest = rest[1:]
+	}
+	for len(rest) > 0 {
+		// Each component is a decimal number followed by a unit. Only the leading sign is allowed.
+		numberEnd := 0
+		for numberEnd < len(rest) && (isDigit(rest[numberEnd]) || rest[numberEnd] == '.') {
+			numberEnd++
+		}
+		unitEnd := numberEnd
+		for unitEnd < len(rest) && !isDigit(rest[unitEnd]) && rest[unitEnd] != '.' {
+			unitEnd++
+		}
+		if numberEnd == 0 || unitEnd == numberEnd {
+			return 0, fmt.Errorf("time: invalid duration '%s'", value)
+		}
+		number, unit := rest[:numberEnd], rest[numberEnd:unitEnd]
+		if unit == "d" {
+			// Use exact decimal arithmetic to avoid float rounding and retain nanosecond precision.
+			days, ok := new(big.Rat).SetString(number)
+			if !ok {
+				return 0, fmt.Errorf("time: invalid duration '%s'", value)
+			}
+			days.Mul(days, new(big.Rat).SetInt64(int64(24*time.Hour)))
+			nanoseconds := new(big.Int).Quo(days.Num(), days.Denom())
+			normalised.WriteString(nanoseconds.String())
+			normalised.WriteString("ns")
+		} else {
+			normalised.WriteString(rest[:unitEnd])
+		}
+		rest = rest[unitEnd:]
+	}
+
+	// The standard parser validates the remaining units and checks the total for overflow.
+	duration, err := time.ParseDuration(normalised.String())
+	if err != nil {
+		return 0, fmt.Errorf("time: invalid duration '%s': %w", value, err)
+	}
+	return duration, nil
+}
+
+func isDigit(c byte) bool {
+	return c >= '0' && c <= '9'
+}

internal/cli/logs/duration_test.go

@@ -0,0 +1,62 @@
+package logs
+
+import (
+	"math"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseDuration(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		input string
+		want  time.Duration
+	}{
+		{"0", 0},
+		{"2m30s", 150 * time.Second},
+		{"1µs", time.Microsecond},
+		{"1μs", time.Microsecond},
+		{"2d", 48 * time.Hour},
+		{"010d", 240 * time.Hour},
+		{"08d", 192 * time.Hour},
+		{"2d3h", 51 * time.Hour},
+		{"3h2d", 51 * time.Hour},
+		{"1d1d", 48 * time.Hour},
+		{"1.5d", 36 * time.Hour},
+		{".5d", 12 * time.Hour},
+		{"1.d", 24 * time.Hour},
+		{"-2d3h", -51 * time.Hour},
+		{"+2d", 48 * time.Hour},
+		{"0d", 0},
+		{"0d1ns", time.Nanosecond},
+		{"0.000000000001d", 86 * time.Nanosecond},
+		{"-0.000000000001d", -86 * time.Nanosecond},
+		{"106751d23h47m16.854775807s", time.Duration(math.MaxInt64)},
+		{"-106751d23h47m16.854775808s", time.Duration(math.MinInt64)},
+	} {
+		t.Run(tt.input, func(t *testing.T) {
+			actual, err := ParseDuration(tt.input)
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, actual)
+		})
+	}
+}
+
+func TestParseDuration_Invalid(t *testing.T) {
+	t.Parallel()
+
+	for _, input := range []string{
+		"", "d", "2d3", "1..2d", ".d", "1d-2h", "1d+2h", "1d 2h", "1day", "2D", "1w",
+		"1e2d", "1d2w", "106752d", "-106752d", "106751d23h47m16.854775808s",
+		"-106751d23h47m16.854775809s", "999999999999999999999999999d",
+	} {
+		t.Run(input, func(t *testing.T) {
+			_, err := ParseDuration(input)
+			require.Error(t, err)
+		})
+	}
+}

internal/cli/logs/formatter.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"charm.land/lipgloss/v2"
+	"github.com/charmbracelet/colorprofile"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/psviderski/uncloud/internal/cli/tui"
 	"github.com/psviderski/uncloud/pkg/api"
@@ -24,6 +25,10 @@ type Formatter struct {
 	maxServiceWidth int
 
 	utc bool
+
+	// Cache each stream's terminal profile to avoid detection on every log entry.
+	stdout *colorprofile.Writer
+	stderr *colorprofile.Writer
 }
 
 func NewFormatter(machineNames, serviceNames []string, utc bool) *Formatter {
@@ -50,6 +55,8 @@ func NewFormatter(machineNames, serviceNames []string, utc bool) *Formatter {
 		maxMachineWidth: maxMachineWidth,
 		maxServiceWidth: maxServiceWidth,
 		utc:             utc,
+		stdout:          colorprofile.NewWriter(os.Stdout, os.Environ()),
+		stderr:          colorprofile.NewWriter(os.Stderr, os.Environ()),
 	}
 }
 
@@ -138,9 +145,9 @@ func (f *Formatter) PrintEntry(entry api.ServiceLogEntry) {
 
 	// Print to appropriate stream.
 	if entry.Stream == api.LogStreamStderr {
-		fmt.Fprint(os.Stderr, output.String())
+		fmt.Fprint(f.stderr, output.String())
 	} else {
-		fmt.Print(output.String())
+		fmt.Fprint(f.stdout, output.String())
 	}
 }
 
@@ -148,8 +155,7 @@ func (f *Formatter) PrintEntry(entry api.ServiceLogEntry) {
 func (f *Formatter) printError(entry api.ServiceLogEntry) {
 	if entry.Metadata.ServiceName == "" {
 		msg := fmt.Sprintf("ERROR: %v", entry.Err)
-		style := lipgloss.NewStyle().Bold(true).Foreground(lipgloss.BrightRed)
-		fmt.Fprintln(os.Stderr, style.Render(msg))
+		fmt.Fprintln(f.stderr, tui.BoldRed.Render(msg))
 		return
 	}
 
@@ -171,8 +177,7 @@ func (f *Formatter) printError(entry api.ServiceLogEntry) {
 		msg += fmt.Sprintf(": %v", entry.Err)
 	}
 
-	style := lipgloss.NewStyle().Bold(true).Foreground(lipgloss.BrightYellow)
-	fmt.Fprintln(os.Stderr, style.Render(msg))
+	fmt.Fprintln(f.stderr, tui.BoldYellow.Render(msg))
 }
 
 // palette is available colors for machine/service differentiation.

internal/cli/logs/logs.go

@@ -19,6 +19,18 @@ type Options struct {
 	Machines []string
 }
 
+// TailLines resolves the default tail limit after parsing flags. An empty opts.Tail means the user
+// did not specify a limit, so --since can select all matching logs without overriding an explicit --tail.
+func (opts Options) TailLines() (int, error) {
+	if opts.Tail == "" {
+		if opts.Since != "" {
+			return -1, nil
+		}
+		return 100, nil
+	}
+	return ParseTail(opts.Tail)
+}
+
 func Flags(options *Options) *pflag.FlagSet {
 	set := &pflag.FlagSet{}
 
@@ -29,14 +41,17 @@ func Flags(options *Options) *pflag.FlagSet {
 	set.StringVar(&options.Since, "since", "",
 		"Show logs generated on or after the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.\n"+
 			"Examples:\n"+
-			"  --since 2m30s                      Relative duration (2 minutes 30 seconds ago)\n"+
-			"  --since 1h                         Relative duration (1 hour ago)\n"+
-			"  --since 2025-11-24                 RFC 3339 date only (midnight using local timezone)\n"+
-			"  --since 2024-05-14T22:50:00        RFC 3339 date/time using local timezone\n"+
+			"  --since 1h45m                      Relative duration (1 hour 45 minutes ago)\n"+
+			"                                     Supported units: d (day = 24h), h (hour), m (minute),\n"+
+			"                                                      s (second), ms (millisecond),\n"+
+			"                                                      us/µs (microsecond), ns (nanosecond)\n"+
+			"  --since 2025-11-24                 RFC 3339 date only (midnight using client local timezone)\n"+
+			"  --since 2024-05-14T22:50:00        RFC 3339 date/time using client local timezone\n"+
 			"  --since 2024-01-31T10:30:00Z       RFC 3339 date/time in UTC\n"+
 			"  --since 1763953966                 Unix timestamp (seconds since January 1, 1970)")
-	set.StringVarP(&options.Tail, "tail", "n", "100",
-		"Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs.")
+	set.StringVarP(&options.Tail, "tail", "n", "",
+		"Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs.\n"+
+			"Defaults to 100, or 'all' when --since is set.")
 	set.StringVar(&options.Until, "until", "",
 		"Show logs generated before the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.\n"+
 			"See --since for examples.")
@@ -46,7 +61,7 @@ func Flags(options *Options) *pflag.FlagSet {
 	return set
 }
 
-func Tail(tail string) (int, error) {
+func ParseTail(tail string) (int, error) {
 	if tail == "all" {
 		return -1, nil
 	}

internal/cli/logs/logs_test.go

@@ -7,6 +7,42 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestOptionsTailLines(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		args []string
+		want int
+	}{
+		{"default", nil, 100},
+		{"follow", []string{"-f"}, 100},
+		{"since", []string{"--since", "1h"}, -1},
+		{"since and follow", []string{"--since", "1h", "-f"}, -1},
+		{"until only", []string{"--until", "1h"}, 100},
+		{"time range", []string{"--since", "3h", "--until", "1h"}, -1},
+		{"explicit default with since", []string{"--since", "1h", "--tail", "100"}, 100},
+		{"explicit limit with since", []string{"--since", "1h", "-n", "20"}, 20},
+		{"explicit limit before since", []string{"-n", "20", "--since", "1h"}, 20},
+		{"explicit all", []string{"--tail", "all"}, -1},
+		{"explicit zero with since", []string{"--since", "1h", "-n", "0", "-f"}, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var options Options
+			require.NoError(t, Flags(&options).Parse(tt.args))
+			tail, err := options.TailLines()
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, tail)
+		})
+	}
+
+	var options Options
+	require.NoError(t, Flags(&options).Parse([]string{"--since", "1h", "--tail", "invalid"}))
+	_, err := options.TailLines()
+	require.ErrorContains(t, err, "invalid --tail value")
+}
+
 func TestParseServiceArgs(t *testing.T) {
 	t.Parallel()
 

internal/cli/logs/time.go

@@ -0,0 +1,72 @@
+package logs
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// TimeRange resolves log filters that could be RFC 3339, timestamp, or relative duration to UTC timestamps.
+// Dates without a timezone use now's location. Relative durations are computed from now.
+func TimeRange(since, until string, now time.Time) (string, string, error) {
+	var err error
+	since, err = timestamp(since, now)
+	if err != nil {
+		return "", "", fmt.Errorf("invalid --since value: %w", err)
+	}
+	until, err = timestamp(until, now)
+	if err != nil {
+		return "", "", fmt.Errorf("invalid --until value: %w", err)
+	}
+
+	return since, until, nil
+}
+
+func timestamp(value string, now time.Time) (string, error) {
+	if value == "" {
+		return "", nil
+	}
+	// A bare zero is the Unix epoch, matching Docker's log filters.
+	if duration, err := ParseDuration(value); value != "0" && err == nil {
+		return now.Add(-duration).UTC().Format(time.RFC3339Nano), nil
+	}
+
+	// Keep Docker's supported date layouts, but use the location's offset at the requested date.
+	for _, layout := range []string{
+		time.RFC3339Nano,
+		"2006-01-02T15:04Z07:00",
+		"2006-01-02T15Z07:00",
+		"2006-01-02Z07:00",
+		"2006-01-02T15:04:05.999999999",
+		"2006-01-02T15:04",
+		"2006-01-02T15",
+		"2006-01-02",
+	} {
+		if t, err := time.ParseInLocation(layout, value, now.Location()); err == nil {
+			return t.UTC().Format(time.RFC3339Nano), nil
+		}
+	}
+
+	seconds, fraction, hasFraction := strings.Cut(value, ".")
+	sec, err := strconv.ParseInt(seconds, 10, 64)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse '%s' as a time or duration", value)
+	}
+
+	var nsec int64
+	if hasFraction {
+		if len(fraction) == 0 || len(fraction) > 9 || strings.ContainsAny(fraction, "+-") {
+			return "", fmt.Errorf("invalid Unix timestamp fraction in '%s'", value)
+		}
+		nsec, err = strconv.ParseInt(fraction+strings.Repeat("0", 9-len(fraction)), 10, 64)
+		if err != nil {
+			return "", fmt.Errorf("invalid Unix timestamp fraction in '%s': %w", value, err)
+		}
+	}
+	t := time.Unix(sec, nsec).UTC()
+	if t.Year() < 0 || t.Year() > 9999 {
+		return "", fmt.Errorf("Unix timestamp '%s' is outside the RFC 3339 date range", value)
+	}
+	return t.Format(time.RFC3339Nano), nil
+}

internal/cli/logs/time_test.go

@@ -0,0 +1,87 @@
+package logs
+
+import (
+	"testing"
+	"time"
+
+	timetypes "github.com/docker/docker/api/types/time"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTimeRange(t *testing.T) {
+	t.Parallel()
+
+	location, err := time.LoadLocation("Australia/Sydney")
+	require.NoError(t, err)
+	// October is daylight-saving time, but July timestamps must use the winter offset.
+	// Daylight saving starts on 4 October. Day durations must still mean 24 elapsed hours.
+	now := time.Date(2026, 10, 5, 12, 0, 0, 123456789, location)
+	tests := []struct {
+		input string
+		want  string
+	}{
+		{"", ""},
+		{"2026-07-01", "2026-06-30T14:00:00Z"},
+		{"2026-07-01T10", "2026-07-01T00:00:00Z"},
+		{"2026-07-01T10:30", "2026-07-01T00:30:00Z"},
+		{"2026-07-01T10:30:45.123456789", "2026-07-01T00:30:45.123456789Z"},
+		{"2026-01-01T10:00:00", "2025-12-31T23:00:00Z"},
+		{"2026-07-01T10:30:45Z", "2026-07-01T10:30:45Z"},
+		{"2026-07-01T10:30:45+02:00", "2026-07-01T08:30:45Z"},
+		{"2026-07-01T10:30:45-04:00", "2026-07-01T14:30:45Z"},
+		{"2026-07-01T10+02:00", "2026-07-01T08:00:00Z"},
+		{"2026-07-01T10:30+02:00", "2026-07-01T08:30:00Z"},
+		{"2026-07-01+02:00", "2026-06-30T22:00:00Z"},
+		{"1763953966", "2025-11-24T03:12:46Z"},
+		{"1763953966.000000001", "2025-11-24T03:12:46.000000001Z"},
+		{"0", "1970-01-01T00:00:00Z"},
+		{"2m30s", "2026-10-05T00:57:30.123456789Z"},
+		{"1d", "2026-10-04T01:00:00.123456789Z"},
+		{"2d", "2026-10-03T01:00:00.123456789Z"},
+		{"2d3h", "2026-10-02T22:00:00.123456789Z"},
+		{"1.5d", "2026-10-03T13:00:00.123456789Z"},
+		{"-2d", "2026-10-07T01:00:00.123456789Z"},
+		{"-1h", "2026-10-05T02:00:00.123456789Z"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			since, until, err := TimeRange(tt.input, tt.input, now)
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, since)
+			assert.Equal(t, tt.want, until)
+
+			if since != "" {
+				// The server-side Docker SDK must preserve the cutoff even in another timezone.
+				actual, err := timetypes.GetTimestamp(since, now.In(time.UTC))
+				require.NoError(t, err)
+				expected, err := time.Parse(time.RFC3339Nano, tt.want)
+				require.NoError(t, err)
+				sec, nsec, err := timetypes.ParseTimestamps(actual, 0)
+				require.NoError(t, err)
+				assert.True(t, expected.Equal(time.Unix(sec, nsec)))
+			}
+		})
+	}
+
+	since, until, err := TimeRange("3h", "1h30m", now)
+	require.NoError(t, err)
+	assert.Equal(t, "2026-10-04T22:00:00.123456789Z", since)
+	assert.Equal(t, "2026-10-04T23:30:00.123456789Z", until)
+}
+
+func TestTimeRange_Invalid(t *testing.T) {
+	t.Parallel()
+
+	for _, input := range []string{
+		"invalid", "2026-02-30", "2026-01-01T25:00:00", "1763953966.xyz",
+		"1763953966.1234567890", "253402300800",
+	} {
+		t.Run(input, func(t *testing.T) {
+			_, _, err := TimeRange(input, "", time.Now())
+			require.ErrorContains(t, err, "invalid --since value")
+			_, _, err = TimeRange("", input, time.Now())
+			require.ErrorContains(t, err, "invalid --until value")
+		})
+	}
+}

internal/cli/tui/print.go

@@ -3,9 +3,11 @@ package tui
 import (
 	"fmt"
 	"os"
+
+	"charm.land/lipgloss/v2"
 )
 
 func PrintWarning(msg string) {
 	styledMsg := BoldYellow.Render(fmt.Sprintf("WARNING: %s", msg))
-	fmt.Fprintln(os.Stderr, styledMsg)
+	lipgloss.Fprintln(os.Stderr, styledMsg)
 }

internal/journal/journal.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os/exec"
+	"time"
 
 	"github.com/psviderski/uncloud/pkg/api"
 )
@@ -31,11 +32,11 @@ func logs(ctx context.Context, unit string, opts api.ServiceLogsOptions) (io.Rea
 
 	if opts.Since != "" {
 		args = append(args, "-S")
-		args = append(args, opts.Since)
+		args = append(args, journalTimestamp(opts.Since))
 	}
 	if opts.Until != "" {
 		args = append(args, "-U")
-		args = append(args, opts.Until)
+		args = append(args, journalTimestamp(opts.Until))
 	}
 
 	cmd := commandContext(ctx, journalctl, args...)
@@ -51,6 +52,18 @@ func logs(ctx context.Context, unit string, opts api.ServiceLogsOptions) (io.Rea
 	return p, cmd.Wait, nil
 }
 
+// journalTimestamp formats normalised client timestamps for journalctl versions that do not
+// accept RFC 3339 timezone suffixes. Support for timestamps containing T and Z was added in
+// systemd 255 (6 December 2023). Debian 12 ships systemd 252, so it still needs this conversion.
+// Journald timestamps have microsecond precision.
+func journalTimestamp(value string) string {
+	if t, err := time.Parse(time.RFC3339Nano, value); err == nil {
+		return t.UTC().Format("2006-01-02 15:04:05.999999") + " UTC"
+	}
+	// Preserve raw filters from older clients and SDK callers.
+	return value
+}
+
 // follow synchronously follows the io.Reader, writing each new journal entry to channel.
 // It stops when the reader is exhausted or the context is cancelled.
 func follow(ctx context.Context, reader io.Reader, outCh chan api.LogEntry) {

internal/journal/logs_test.go

@@ -115,6 +115,9 @@ func TestEntry(t *testing.T) {
 }
 
 func TestLogs(t *testing.T) {
+	originalCommandContext := commandContext
+	t.Cleanup(func() { commandContext = originalCommandContext })
+
 	commandContext = func(ctx context.Context, _ string, _ ...string) *exec.Cmd {
 		return exec.CommandContext(ctx, "/usr/bin/tail", "testdata/logs")
 	}
@@ -150,3 +153,38 @@ func TestLogs(t *testing.T) {
 	// Still six because heartbeats are not written here and Tail is ignored as the command is overridden.
 	assert.Equal(t, 6, i)
 }
+
+func TestLogs_TimeFilters(t *testing.T) {
+	originalCommandContext := commandContext
+	t.Cleanup(func() { commandContext = originalCommandContext })
+
+	var args []string
+	commandContext = func(ctx context.Context, command string, commandArgs ...string) *exec.Cmd {
+		assert.Equal(t, "journalctl", command)
+		args = commandArgs
+		return exec.CommandContext(ctx, "/usr/bin/tail", "testdata/logs")
+	}
+
+	ch, err := Logs(context.Background(), "uncloud", api.ServiceLogsOptions{
+		Since: "2026-07-01T10:30:45.123456789+10:00",
+		Until: "2026-07-01T01:30:45Z",
+	})
+	require.NoError(t, err)
+	for range ch {
+	}
+	assert.Equal(t, []string{
+		"-u", "uncloud", "--no-hostname", "-n", "0", "-o", "short-unix",
+		"-S", "2026-07-01 00:30:45.123456 UTC",
+		"-U", "2026-07-01 01:30:45 UTC",
+	}, args)
+
+	// Older clients and SDK callers can still pass journalctl's native filters.
+	ch, err = Logs(context.Background(), "uncloud", api.ServiceLogsOptions{Since: "1h ago", Until: "today"})
+	require.NoError(t, err)
+	for range ch {
+	}
+	assert.Equal(t, []string{
+		"-u", "uncloud", "--no-hostname", "-n", "0", "-o", "short-unix",
+		"-S", "1h ago", "-U", "today",
+	}, args)
+}

internal/machine/caddyconfig/mocks_test.go

@@ -16,10 +16,19 @@ func NewMockCaddyfileValidator(t interface {
 	mock.TestingT
 	Cleanup(func())
 }) *MockCaddyfileValidator {
+	if helper, ok := t.(interface{ Helper() }); ok {
+		helper.Helper()
+	}
+
 	mock := &MockCaddyfileValidator{}
 	mock.Mock.Test(t)
 
-	t.Cleanup(func() { mock.AssertExpectations(t) })
+	t.Cleanup(func() {
+		if helper, ok := t.(interface{ Helper() }); ok {
+			helper.Helper()
+		}
+		mock.AssertExpectations(t)
+	})
 
 	return mock
 }

internal/machine/cluster/cluster.go

@@ -14,6 +14,7 @@ import (
 	"github.com/psviderski/uncloud/internal/machine/network"
 	"github.com/psviderski/uncloud/internal/machine/store"
 	"github.com/psviderski/uncloud/internal/secret"
+	"github.com/psviderski/uncloud/pkg/api"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/emptypb"
@@ -87,6 +88,11 @@ func (c *Cluster) AddMachine(ctx context.Context, req *pb.AddMachineRequest) (*p
 func (c *Cluster) AddMachineWithoutReadyCheck(
 	ctx context.Context, req *pb.AddMachineRequest,
 ) (*pb.AddMachineResponse, error) {
+	if req.Name != "" {
+		if err := api.ValidateMachineName(req.Name); err != nil {
+			return nil, status.Error(codes.InvalidArgument, err.Error())
+		}
+	}
 	if req.Network == nil {
 		return nil, status.Error(codes.InvalidArgument, "network not set")
 	}

internal/machine/cluster/cluster_test.go

@@ -0,0 +1,28 @@
+package cluster
+
+import (
+	"context"
+	"testing"
+
+	"github.com/psviderski/uncloud/api/pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func TestAddMachine_InvalidName(t *testing.T) {
+	t.Parallel()
+
+	ready := make(chan struct{})
+	close(ready)
+	// Invalid names must be rejected before any store access.
+	c := NewCluster(nil, nil, nil, ready)
+	for _, name := range []string{"VPS1", "worker.example", "rr", "nearest", "c337f00600de51ef4375c9a9a267dba5"} {
+		t.Run("add", func(t *testing.T) {
+			resp, err := c.AddMachine(context.Background(), &pb.AddMachineRequest{Name: name})
+			require.Nil(t, resp)
+			require.Equal(t, codes.InvalidArgument, status.Code(err))
+			require.ErrorContains(t, err, "invalid machine name")
+		})
+	}
+}

internal/machine/cluster/machine.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/psviderski/uncloud/internal/secret"
+	"github.com/psviderski/uncloud/pkg/api"
 )
 
 // NewMachineID generates a new unique machine ID.
@@ -27,7 +28,7 @@ func NewRandomMachineName() (string, error) {
 // a numeric suffix ("-1", "-2", etc.) if needed.
 func DefaultMachineName(hostname string, existing []string) (string, error) {
 	name := machineNameFromHostname(hostname)
-	if name == "" {
+	if api.ValidateMachineName(name) != nil {
 		var err error
 		if name, err = NewRandomMachineName(); err != nil {
 			return "", err
@@ -38,7 +39,9 @@ func DefaultMachineName(hostname string, existing []string) (string, error) {
 		return name, nil
 	}
 	for i := 1; ; i++ {
-		candidate := fmt.Sprintf("%s-%d", name, i)
+		suffix := fmt.Sprintf("-%d", i)
+		base := strings.TrimRight(name[:min(len(name), 63-len(suffix))], "-")
+		candidate := base + suffix
 		if !slices.Contains(existing, candidate) {
 			return candidate, nil
 		}
@@ -55,12 +58,13 @@ func machineNameFromHostname(hostname string) string {
 	var b strings.Builder
 	for _, r := range label {
 		switch {
-		case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_':
+		case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-':
 			b.WriteRune(r)
 		default:
 			b.WriteRune('-')
 		}
 	}
 	// Trim leading and trailing hyphens that may result from the sanitisation above.
-	return strings.Trim(b.String(), "-")
+	name := strings.Trim(b.String(), "-")
+	return strings.TrimRight(name[:min(len(name), 63)], "-")
 }

internal/machine/cluster/machine_test.go

@@ -1,8 +1,10 @@
 package cluster
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -18,9 +20,11 @@ func TestMachineNameFromHostname(t *testing.T) {
 		{"simple", "web", "web"},
 		{"fqdn uses first label", "web-1.example.com", "web-1"},
 		{"uppercase lowercased", "Web-Server", "web-server"},
-		{"invalid chars replaced", "host_name@1", "host_name-1"},
-		{"trim surrounding hyphens", "-_host_-", "_host_"},
+		{"invalid chars replaced", "host_name@1", "host-name-1"},
+		{"trim surrounding hyphens", "-_host_-", "host"},
 		{"whitespace trimmed", "  myhost  ", "myhost"},
+		{"long hostname truncated", strings.Repeat("a", 64), strings.Repeat("a", 63)},
+		{"truncation trims trailing hyphen", strings.Repeat("a", 62) + "-b", strings.Repeat("a", 62)},
 		{"empty", "", ""},
 		{"only invalid chars", "@#", ""},
 		{"dot only", ".example.com", ""},
@@ -37,10 +41,12 @@ func TestDefaultMachineName(t *testing.T) {
 	t.Parallel()
 
 	t.Run("falls back to random", func(t *testing.T) {
-		// An empty or fully invalid hostname falls back to a random "machine-xxxx" name.
-		got, err := DefaultMachineName("***", nil)
-		require.NoError(t, err)
-		assert.Regexp(t, `^machine-[a-zA-Z0-9]{4}$`, got)
+		for _, hostname := range []string{"***", "rr", "nearest", strings.Repeat("a", 32)} {
+			got, err := DefaultMachineName(hostname, nil)
+			require.NoError(t, err)
+			assert.Regexp(t, `^machine-[a-z0-9]{4}$`, got)
+			require.NoError(t, api.ValidateMachineName(got))
+		}
 	})
 
 	tests := []struct {
@@ -49,10 +55,45 @@ func TestDefaultMachineName(t *testing.T) {
 		existing []string
 		want     string
 	}{
-		{"from hostname", "web-1.example.com", nil, "web-1"},
-		{"dedup against existing", "web", []string{"web"}, "web-1"},
-		{"dedup multiple", "web", []string{"web", "web-1"}, "web-2"},
-		{"sanitized", "My_Host", nil, "my_host"},
+		{
+			name:     "from hostname",
+			hostname: "web-1.example.com",
+			want:     "web-1",
+		},
+		{
+			name:     "dedup against existing",
+			hostname: "web",
+			existing: []string{"web"},
+			want:     "web-1",
+		},
+		{
+			name:     "dedup multiple",
+			hostname: "web",
+			existing: []string{"web", "web-1"},
+			want:     "web-2",
+		},
+		{
+			name:     "sanitized",
+			hostname: "My_Host",
+			want:     "my-host",
+		},
+		{
+			name:     "long hostname",
+			hostname: strings.Repeat("a", 64),
+			want:     strings.Repeat("a", 63),
+		},
+		{
+			name:     "dedup maximum length",
+			hostname: strings.Repeat("a", 63),
+			existing: []string{strings.Repeat("a", 63)},
+			want:     strings.Repeat("a", 61) + "-1",
+		},
+		{
+			name:     "dedup trims trailing hyphen",
+			hostname: strings.Repeat("a", 60) + "-bb",
+			existing: []string{strings.Repeat("a", 60) + "-bb"},
+			want:     strings.Repeat("a", 60) + "-1",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -60,6 +101,7 @@ func TestDefaultMachineName(t *testing.T) {
 			got, err := DefaultMachineName(tt.hostname, tt.existing)
 			require.NoError(t, err)
 			assert.Equal(t, tt.want, got)
+			require.NoError(t, api.ValidateMachineName(got))
 		})
 	}
 }

internal/machine/dns/resolver.go

@@ -10,6 +10,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/psviderski/uncloud/api/pb"
+	"github.com/psviderski/uncloud/internal/machine/network"
 	"github.com/psviderski/uncloud/internal/machine/store"
 )
 
@@ -19,7 +21,9 @@ type ClusterResolver struct {
 	store *store.Store
 	// serviceIPs maps service names to container IPs.
 	serviceIPs map[string][]netip.Addr
-	// mu protects the serviceIPs map.
+	// machineIPs maps machine IDs and names to their IP.
+	machineIPs map[string]netip.Addr
+	// mu protects the serviceIPs and machineIPs map.
 	mu sync.RWMutex
 	// lastUpdate tracks when records were last updated.
 	lastUpdate time.Time
@@ -46,6 +50,13 @@ func (r *ClusterResolver) Run(ctx context.Context) error {
 	// TODO: implement machine membership check using Corrossion Admin client to filter available containers.
 	r.updateServiceIPs(containers)
 
+	machines, mchanges, err := r.store.SubscribeMachines(ctx)
+	if err != nil {
+		return fmt.Errorf("subscribe to machine changes: %w", err)
+	}
+	r.log.Info("Subscribed to machine changes in the clsuter to keep machine DNS records updated.")
+	r.updateMachineIPs(machines)
+
 	for {
 		select {
 		case _, ok := <-changes:
@@ -59,9 +70,22 @@ func (r *ClusterResolver) Run(ctx context.Context) error {
 				r.log.Error("Failed to list containers.", "err", err)
 				continue
 			}
-
 			// TODO: implement machine membership check using Corrossion Admin client to filter available containers.
 			r.updateServiceIPs(containers)
+
+		case _, ok := <-mchanges:
+			if !ok {
+				return fmt.Errorf("machine subscription failed")
+			}
+			r.log.Debug("Cluster machines changed, updating DNS records.")
+
+			machines, err := r.store.ListMachines(ctx)
+			if err != nil {
+				r.log.Error("Failed to list machines.", "err", err)
+				continue
+			}
+			r.updateMachineIPs(machines)
+
 		case <-ctx.Done():
 			return nil
 		}
@@ -121,19 +145,49 @@ func (r *ClusterResolver) updateServiceIPs(containers []store.ContainerRecord) {
 	r.log.Info("DNS records updated.", "services", len(newServiceIPs)/3, "containers", containersCount)
 }
 
-// Resolve returns IP addresses of the service containers.
-func (r *ClusterResolver) Resolve(serviceName string) []netip.Addr {
+func (r *ClusterResolver) updateMachineIPs(machines []*pb.MachineInfo) {
+	newMachineIPs := make(map[string]netip.Addr, len(machines))
+
+	for _, machine := range machines {
+		subnet, err := machine.Network.Subnet.ToPrefix()
+		if err != nil {
+			continue
+		}
+		addr := network.MachineIP(subnet)
+		newMachineIPs[machine.Name+".m"] = addr
+		newMachineIPs[machine.Id+".m"] = addr
+	}
+	r.mu.Lock()
+	r.machineIPs = newMachineIPs
+	r.mu.Unlock()
+
+	r.log.Info("DNS records updated.", "machines", len(machines))
+}
+
+// Resolve returns IP addresses of the service containers or machines.
+func (r *ClusterResolver) Resolve(name string) []netip.Addr {
 	r.mu.RLock()
 	defer r.mu.RUnlock()
 
-	ips, ok := r.serviceIPs[serviceName]
-	if !ok || len(ips) == 0 {
-		return nil
+	// Return a copy of the IPs slice to prevent modification of the original.
+	ips, ok := r.serviceIPs[name]
+	if ok && len(ips) > 0 {
+		return slices.Clone(ips)
 	}
 
-	// Return a copy of the IPs slice to prevent modification of the original.
-	ipsCopy := make([]netip.Addr, len(ips))
-	copy(ipsCopy, ips)
+	ip, ok := r.machineIPs[name]
+	if ok {
+		return slices.Clone([]netip.Addr{ip})
+	}
 
-	return ipsCopy
+	if name == "m" { // all machines subdomain
+		// collect all unique IP address
+		uniq := map[string]netip.Addr{}
+		for _, addr := range r.machineIPs {
+			uniq[addr.String()] = addr
+		}
+		return slices.Clone(slices.Collect(maps.Values(uniq)))
+	}
+
+	return nil
 }

internal/machine/dns/resolver_test.go

@@ -1,11 +1,13 @@
 package dns
 
 import (
+	"net/netip"
 	"reflect"
 	"testing"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
+	"github.com/psviderski/uncloud/api/pb"
 	"github.com/psviderski/uncloud/internal/machine/store"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/stretchr/testify/assert"
@@ -75,3 +77,32 @@ func newRecord(serviceID, serviceName, ip, machineID string) store.ContainerReco
 		MachineID: machineID,
 	}
 }
+
+func TestClusterResolver_UpdateMachineIPs(t *testing.T) {
+	t.Parallel()
+
+	machines := []*pb.MachineInfo{
+		newMachineRecord("x0y0z0", "mach-1", "10.210.0.0/24"),
+		newMachineRecord("x1y1z1", "mach-2", "10.210.1.0/24"),
+		newMachineRecord("x2y2z2", "mach-3", "10.210.2.0/24"),
+	}
+
+	r := NewClusterResolver(nil)
+	r.updateMachineIPs(machines)
+
+	assert.NotEmpty(t, r.Resolve("mach-1.m"))
+	assert.NotEmpty(t, r.Resolve("mach-3.m"))
+	assert.NotEmpty(t, r.Resolve("x0y0z0.m"))
+
+	assert.Equal(t, 3, len(r.Resolve("m")))
+}
+
+func newMachineRecord(machineID, machineName, prefix string) *pb.MachineInfo {
+	return &pb.MachineInfo{
+		Id:   machineID,
+		Name: machineName,
+		Network: &pb.NetworkConfig{
+			Subnet: pb.NewIPPrefix(netip.MustParsePrefix(prefix)),
+		},
+	}
+}

internal/machine/dns/server.go

@@ -293,13 +293,13 @@ func (s *Server) forwardRequest(req *dns.Msg, proto string) (*dns.Msg, error) {
 // handleAQuery processes an A query for the internal domain and returns A records for the requested name.
 // The internal domain suffix is already stripped from the name. An empty list is returned if no records are found.
 func (s *Server) handleAQuery(name string) []dns.RR {
-	serviceName, mode := extractModeFromDomain(trimInternalDomain(name))
-	ips := s.resolver.Resolve(serviceName)
+	unname, mode := extractModeFromDomain(trimInternalDomain(name))
+	ips := s.resolver.Resolve(unname)
 	if len(ips) == 0 {
-		s.log.Debug("Failed to resolve service name.", "service", serviceName)
+		s.log.Debug("Failed to resolve internal name.", "name", name)
 		return nil
 	}
-	s.log.Debug("Resolved service name.", "service", serviceName, "ips", ips)
+	s.log.Debug("Resolved service internal name.", "name", name, "ips", ips)
 
 	if len(ips) > 1 {
 		// Shuffle the IPs to approximate round-robin.

internal/machine/docker/client.go

@@ -167,7 +167,7 @@ func (c *Client) ListContainers(ctx context.Context, opts container.ListOptions)
 	machineContainers := make([]MachineContainers, len(resp.Messages))
 	for i, msg := range resp.Messages {
 		machineContainers[i].Metadata = msg.Metadata
-		if msg.Metadata != nil && msg.Metadata.Error != "" {
+		if msg.Metadata.Err() != nil {
 			continue
 		}
 
@@ -258,7 +258,7 @@ func (c *Client) InspectImage(ctx context.Context, id string) ([]api.MachineImag
 
 	notFoundCount := 0
 	for _, msg := range resp.Messages {
-		if msg.Metadata != nil && msg.Metadata.Status != nil && codes.Code(msg.Metadata.Status.Code) == codes.NotFound {
+		if status.Code(msg.Metadata.Err()) == codes.NotFound {
 			notFoundCount++
 		}
 	}
@@ -269,7 +269,7 @@ func (c *Client) InspectImage(ctx context.Context, id string) ([]api.MachineImag
 	images := make([]api.MachineImage, len(resp.Messages))
 	for i, msg := range resp.Messages {
 		images[i].Metadata = msg.Metadata
-		if msg.Metadata != nil && msg.Metadata.Error != "" {
+		if msg.Metadata.Err() != nil {
 			continue
 		}
 
@@ -312,7 +312,7 @@ func parseRemoteImageMessage(msg *pb.RemoteImage) (api.MachineRemoteImage, error
 	mri := api.MachineRemoteImage{
 		Metadata: msg.Metadata,
 	}
-	if msg.Metadata != nil && msg.Metadata.Error != "" {
+	if msg.Metadata.Err() != nil {
 		return mri, nil
 	}
 
@@ -393,7 +393,7 @@ func (c *Client) ListVolumes(ctx context.Context, opts volume.ListOptions) ([]Ma
 	machineVolumes := make([]MachineVolumes, len(resp.Messages))
 	for i, msg := range resp.Messages {
 		machineVolumes[i].Metadata = msg.Metadata
-		if msg.Metadata != nil && msg.Metadata.Error != "" {
+		if msg.Metadata.Err() != nil {
 			continue
 		}
 
@@ -472,7 +472,7 @@ func (c *Client) ListServiceContainers(
 	machineContainers := make([]MachineServiceContainers, len(resp.Messages))
 	for i, msg := range resp.Messages {
 		machineContainers[i].Metadata = msg.Metadata
-		if msg.Metadata != nil && msg.Metadata.Error != "" {
+		if msg.Metadata.Err() != nil {
 			continue
 		}
 

internal/machine/docker/controller.go

@@ -27,7 +27,7 @@ const (
 type Controller struct {
 	// machineID is the ID of the machine where the managed Docker daemon is running.
 	machineID string
-	client    *client.Client
+	client    client.APIClient
 	service   *Service
 	store     *store.Store
 }

internal/machine/docker/image.go

@@ -0,0 +1,79 @@
+package docker
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/containerd/errdefs"
+	"github.com/distribution/reference"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/image"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/psviderski/uncloud/pkg/api"
+)
+
+// RemoveImage removes an image while protecting names used by containers, even when force is requested.
+func (s *Service) RemoveImage(ctx context.Context, id string, opts image.RemoveOptions) ([]image.DeleteResponse, error) {
+	if err := s.checkImageRemovalConflicts(ctx, id); err != nil {
+		return nil, err
+	}
+	// Docker enforces its own conflicts, including protection of a used image's final reference.
+	return s.Client.ImageRemove(ctx, id, opts)
+}
+
+// checkImageRemovalConflicts rejects an image ID used by a running or stopped container, or a name used by a
+// container while it still resolves to that container's image. A digest request also checks the repository tags
+// Docker would remove with it. Unused aliases are allowed.
+func (s *Service) checkImageRemovalConflicts(ctx context.Context, id string) error {
+	img, err := s.Client.ImageInspect(ctx, id)
+	if err != nil {
+		return err
+	}
+	// Match Docker's resolved ID before parsing names, since a short ID also parses as a repository name.
+	isID := strings.HasPrefix(strings.TrimPrefix(img.ID, "sha256:"), strings.TrimPrefix(id, "sha256:"))
+	var named reference.Named
+	if !isID {
+		named, err = reference.ParseDockerRef(id)
+		if err != nil {
+			return errdefs.ErrInvalidArgument.WithMessage(err.Error())
+		}
+	}
+	_, isDigest := named.(reference.Canonical)
+
+	containers, err := s.Client.ContainerList(ctx, container.ListOptions{
+		All: true, Filters: filters.NewArgs(filters.Arg("ancestor", img.ID)),
+	})
+	if err != nil {
+		return fmt.Errorf("list containers using image: %w", err)
+	}
+
+	for _, ctr := range containers {
+		// The ancestor filter can include descendants. A retagged name must also be checked against its current ID.
+		if ctr.ImageID != img.ID {
+			continue
+		}
+
+		usedRef, err := reference.ParseDockerRef(ctr.Image)
+		// Familiar names make "alpine:latest" and "docker.io/library/alpine:latest" identify the same tag.
+		removesUsedName := err == nil && named != nil && reference.FamiliarString(usedRef) == reference.FamiliarString(named)
+		if isDigest && err == nil && usedRef.Name() == named.Name() {
+			// Removing a digest also removes this repository's tags pointing at the same image.
+			removesUsedName = true
+		}
+		if isID || removesUsedName {
+			ctrID, state := stringid.TruncateID(ctr.ID), "stopped"
+			if serviceName := ctr.Labels[api.LabelServiceName]; serviceName != "" {
+				ctrID = serviceName + "/" + ctrID
+			}
+			if ctr.State == container.StateRunning || ctr.State == container.StatePaused {
+				state = container.StateRunning
+			}
+			return errdefs.ErrConflict.WithMessage(
+				fmt.Sprintf("image '%s' is used by container '%s' (%s)", id, ctrID, state))
+		}
+	}
+
+	return nil
+}

internal/machine/docker/image_test.go

@@ -0,0 +1,175 @@
+package docker
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/containerd/errdefs"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/image"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCheckImageRemovalConflicts(t *testing.T) {
+	imageID := "sha256:" + strings.Repeat("a", 64)
+	otherID := "sha256:" + strings.Repeat("b", 64)
+	digestRef := "myapp@" + imageID
+
+	tests := []struct {
+		name             string
+		requested        string
+		configured       string
+		containerImageID string
+		serviceName      string
+		blocked          bool
+	}{
+		{name: "used tag", requested: "myapp:current", configured: "myapp:current", serviceName: "web", blocked: true},
+		{name: "qualified request", requested: "docker.io/library/myapp:current", configured: "myapp:current", blocked: true},
+		{name: "qualified container reference", requested: "myapp:current", configured: "docker.io/library/myapp:current", blocked: true},
+		{name: "implicit latest", requested: "myapp", configured: "myapp:latest", blocked: true},
+		{name: "unused alias", requested: "myapp:old", configured: "myapp:current"},
+		{name: "unused alias of ID-used image", requested: "myapp:old", configured: imageID},
+		{name: "tag may clean up a used digest", requested: "myapp:old", configured: digestRef},
+		{name: "used digest", requested: digestRef, configured: digestRef, blocked: true},
+		{name: "digest also removes used tag", requested: digestRef, configured: "myapp:current", blocked: true},
+		{name: "unused repository digest", requested: digestRef, configured: "other:keep"},
+		{name: "image ID", requested: imageID, configured: "myapp:current", blocked: true},
+		{name: "short image ID", requested: "aaaaaaaa", configured: "myapp:current", blocked: true},
+		{name: "short image ID with algorithm", requested: "sha256:aaaaaaaa", configured: "myapp:current", blocked: true},
+		{name: "retagged name", requested: "myapp:current", configured: "myapp:current", containerImageID: otherID},
+	}
+
+	for _, tt := range tests {
+		for _, state := range []string{container.StateExited, container.StateRunning} {
+			t.Run(fmt.Sprintf("%s/%s", tt.name, state), func(t *testing.T) {
+				containerImageID := tt.containerImageID
+				if containerImageID == "" {
+					containerImageID = imageID
+				}
+
+				cli := NewMockAPIClient(t)
+				cli.EXPECT().ImageInspect(mock.Anything, tt.requested).
+					Return(image.InspectResponse{ID: imageID}, nil).Once()
+				cli.EXPECT().ContainerList(mock.Anything, mock.MatchedBy(func(opts container.ListOptions) bool {
+					return opts.All // Stopped containers must be checked too.
+				})).Return([]container.Summary{{
+					ID:      strings.Repeat("c", 64),
+					Labels:  map[string]string{api.LabelServiceName: tt.serviceName},
+					Names:   []string{"/web"},
+					ImageID: containerImageID,
+					Image:   tt.configured,
+					State:   state,
+				}}, nil).Once()
+
+				service := NewService(cli, nil)
+				err := service.checkImageRemovalConflicts(context.Background(), tt.requested)
+
+				if tt.blocked {
+					require.True(t, errdefs.IsConflict(err), "%v", err)
+					assert.ErrorContains(t, err, tt.requested)
+					if tt.serviceName != "" {
+						assert.ErrorContains(t, err, "container 'web/cccccccccccc'")
+					} else {
+						assert.ErrorContains(t, err, "container 'cccccccccccc'")
+					}
+					wantState := "stopped"
+					if state == container.StateRunning {
+						wantState = state
+					}
+					assert.ErrorContains(t, err, wantState)
+				} else {
+					require.NoError(t, err)
+				}
+			})
+		}
+	}
+}
+
+func TestCheckImageRemovalConflicts_PropagatesLookupErrors(t *testing.T) {
+	imageID := "sha256:" + strings.Repeat("a", 64)
+	t.Run("image lookup fails", func(t *testing.T) {
+		cli := NewMockAPIClient(t)
+		lookupErr := errdefs.ErrNotFound.WithMessage("no such image")
+		cli.EXPECT().ImageInspect(mock.Anything, "myapp:old").Return(image.InspectResponse{}, lookupErr).Once()
+
+		service := NewService(cli, nil)
+		require.ErrorIs(t, service.checkImageRemovalConflicts(context.Background(), "myapp:old"), lookupErr)
+	})
+
+	t.Run("container lookup fails", func(t *testing.T) {
+		cli := NewMockAPIClient(t)
+		lookupErr := errors.New("Docker unavailable")
+		cli.EXPECT().ImageInspect(mock.Anything, "myapp:old").Return(image.InspectResponse{ID: imageID}, nil).Once()
+		cli.EXPECT().ContainerList(mock.Anything, mock.Anything).Return(nil, lookupErr).Once()
+
+		service := NewService(cli, nil)
+		err := service.checkImageRemovalConflicts(context.Background(), "myapp:old")
+
+		require.ErrorIs(t, err, lookupErr)
+		assert.ErrorContains(t, err, "list containers using image")
+	})
+}
+
+func TestServiceRemoveImage_RejectsUsedNameBeforeDeletion(t *testing.T) {
+	imageID := "sha256:" + strings.Repeat("a", 64)
+	cli := NewMockAPIClient(t)
+	cli.EXPECT().ImageInspect(mock.Anything, "myapp:current").Return(image.InspectResponse{ID: imageID}, nil).Once()
+	cli.EXPECT().ContainerList(mock.Anything, mock.Anything).Return([]container.Summary{{
+		ID: "container-id", ImageID: imageID, Image: "myapp:current", State: container.StateRunning,
+	}}, nil).Once()
+
+	service := NewService(cli, nil)
+	// ImageRemove has no expectation: any deletion call would fail this test.
+	results, err := service.RemoveImage(context.Background(), "myapp:current", image.RemoveOptions{Force: true})
+
+	require.True(t, errdefs.IsConflict(err), "%v", err)
+	assert.Nil(t, results)
+}
+
+func TestServiceRemoveImage_DelegatesToDocker(t *testing.T) {
+	imageID := "sha256:" + strings.Repeat("a", 64)
+	tests := []struct {
+		name    string
+		opts    image.RemoveOptions
+		results []image.DeleteResponse
+		err     error
+	}{
+		{
+			name:    "options and results pass through",
+			opts:    image.RemoveOptions{Force: true, PruneChildren: true, Platforms: []ocispec.Platform{{OS: "linux", Architecture: "amd64"}}},
+			results: []image.DeleteResponse{{Deleted: imageID}},
+		},
+		{
+			name: "Docker rejects final reference removal",
+			opts: image.RemoveOptions{PruneChildren: true},
+			err:  errdefs.ErrConflict.WithMessage("container is using its referenced image"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cli := NewMockAPIClient(t)
+			cli.EXPECT().ImageInspect(mock.Anything, "myapp:old").Return(image.InspectResponse{ID: imageID}, nil).Once()
+			cli.EXPECT().ContainerList(mock.Anything, mock.Anything).Return([]container.Summary{{
+				ID: "container-id", ImageID: imageID, Image: "myapp@" + imageID, State: container.StateRunning,
+			}}, nil).Once()
+			cli.EXPECT().ImageRemove(mock.Anything, "myapp:old", tt.opts).Return(tt.results, tt.err).Once()
+
+			service := NewService(cli, nil)
+			results, err := service.RemoveImage(context.Background(), "myapp:old", tt.opts)
+			if tt.err != nil {
+				require.ErrorIs(t, err, tt.err)
+			} else {
+				require.NoError(t, err)
+			}
+			assert.Equal(t, tt.results, results)
+		})
+	}
+}

internal/machine/docker/server.go

@@ -56,7 +56,7 @@ var fullDockerIDRegex = regexp.MustCompile(`^[a-f0-9]{64}$`)
 // Server implements the gRPC Docker service that proxies requests to the Docker daemon.
 type Server struct {
 	pb.UnimplementedDockerServer
-	client  *client.Client
+	client  client.APIClient
 	service *Service
 	db      *sqlx.DB
 	// internalDNSIP is a function that returns the IP address of the internal DNS server. It may return an empty
@@ -349,6 +349,41 @@ func (s *Server) InspectImage(ctx context.Context, req *pb.InspectImageRequest)
 	}, nil
 }
 
+// RemoveImage removes an image while protecting names used by containers, even when force is requested.
+func (s *Server) RemoveImage(ctx context.Context, req *pb.RemoveImageRequest) (*pb.RemoveImageResponse, error) {
+	opts := image.RemoveOptions{Force: req.Force, PruneChildren: req.PruneChildren}
+	for _, platform := range req.Platforms {
+		opts.Platforms = append(opts.Platforms, ocispec.Platform{
+			Architecture: platform.Architecture,
+			OS:           platform.Os,
+			OSVersion:    platform.OsVersion,
+			OSFeatures:   platform.OsFeatures,
+			Variant:      platform.Variant,
+		})
+	}
+
+	removals, err := s.service.RemoveImage(ctx, req.Id, opts)
+	if err != nil {
+		switch {
+		case errdefs.IsNotFound(err):
+			return nil, status.Error(codes.NotFound, err.Error())
+		case errdefs.IsConflict(err):
+			return nil, status.Error(codes.FailedPrecondition, err.Error())
+		case errdefs.IsInvalidArgument(err):
+			return nil, status.Error(codes.InvalidArgument, err.Error())
+		default:
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	}
+
+	result := &pb.MachineImageRemoval{}
+	for _, removal := range removals {
+		result.Results = append(result.Results, &pb.ImageRemoval{Deleted: removal.Deleted, Untagged: removal.Untagged})
+	}
+
+	return &pb.RemoveImageResponse{Messages: []*pb.MachineImageRemoval{result}}, nil
+}
+
 // InspectRemoteImage returns the image metadata for an image in a remote registry using the machine's Docker auth
 // credentials if necessary.
 func (s *Server) InspectRemoteImage(

internal/machine/docker/service.go

@@ -31,13 +31,13 @@ import (
 // from the machine database.
 type Service struct {
 	// Client is a Docker client for managing Docker resources.
-	Client *client.Client
+	Client client.APIClient
 	// db is a connection to the machine database.
 	db *sqlx.DB
 }
 
 // NewService creates a new Docker service instance.
-func NewService(client *client.Client, db *sqlx.DB) *Service {
+func NewService(client client.APIClient, db *sqlx.DB) *Service {
 	return &Service{
 		Client: client,
 		db:     db,

internal/machine/machine.go

@@ -836,6 +836,11 @@ func (m *Machine) InitCluster(ctx context.Context, req *pb.InitClusterRequest) (
 	if m.Initialised() {
 		return nil, status.Error(codes.FailedPrecondition, "machine is already configured as a cluster member")
 	}
+	if req.MachineName != "" {
+		if err := api.ValidateMachineName(req.MachineName); err != nil {
+			return nil, status.Error(codes.InvalidArgument, err.Error())
+		}
+	}
 
 	clusterNetwork, err := req.Network.ToPrefix()
 	if err != nil {
@@ -1220,8 +1225,8 @@ func (m *Machine) applyMachineUpdate(ctx context.Context, req *pb.UpdateMachineR
 	defer m.state.mu.Unlock()
 
 	if req.Name != nil {
-		if *req.Name == "" {
-			return status.Error(codes.InvalidArgument, "machine name cannot be empty")
+		if err := api.ValidateMachineName(*req.Name); err != nil {
+			return status.Error(codes.InvalidArgument, err.Error())
 		}
 		// Check for duplicate names across the cluster, excluding this machine.
 		if *req.Name != m.state.Name {

internal/machine/machine_test.go

@@ -0,0 +1,38 @@
+package machine
+
+import (
+	"context"
+	"testing"
+
+	"github.com/psviderski/uncloud/api/pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func TestInitCluster_InvalidMachineName(t *testing.T) {
+	t.Parallel()
+
+	for _, name := range []string{"VPS1", "rr", "nearest", "c337f00600de51ef4375c9a9a267dba5"} {
+		// Invalid names must be rejected before initializing cluster state.
+		m := &Machine{state: &State{}}
+
+		resp, err := m.InitCluster(context.Background(), &pb.InitClusterRequest{MachineName: name})
+		require.Nil(t, resp)
+		require.Equal(t, codes.InvalidArgument, status.Code(err))
+		require.ErrorContains(t, err, "invalid machine name")
+	}
+}
+
+func TestUpdateMachine_InvalidName(t *testing.T) {
+	t.Parallel()
+
+	for _, name := range []string{"", "VPS1", "worker.example", "rr", "nearest", "c337f00600de51ef4375c9a9a267dba5"} {
+		m := &Machine{state: &State{ID: "machine-id", Name: "worker"}}
+		resp, err := m.UpdateMachine(context.Background(), &pb.UpdateMachineRequest{Name: &name})
+		require.Nil(t, resp)
+		require.Equal(t, codes.InvalidArgument, status.Code(err))
+		require.ErrorContains(t, err, "invalid machine name")
+		require.Equal(t, "worker", m.state.Name)
+	}
+}

internal/machine/store/container.go

@@ -23,6 +23,10 @@ const (
 	// SyncStatusOutdated indicates that a container record may be outdated, for example, due to being unable
 	// to retrieve the container's state from the Docker daemon or when the machine is being stopped or restarted.
 	SyncStatusOutdated = "outdated"
+
+	// containerChangesDebounceInterval defines how long to wait before notifying subscribers about container changes.
+	// Multiple changes within this window are grouped into a single notification to prevent system overload.
+	containerChangesDebounceInterval = 250 * time.Millisecond
 )
 
 type ContainerRecord struct {
@@ -92,6 +96,9 @@ func normaliseContainerForStore(ctr *api.ServiceContainer) {
 	// Remove the environment variables to avoid leaking secrets.
 	ctr.Config.Env = nil
 	ctr.ServiceSpec.Container.Env = nil
+	if ctr.ServiceSpec.PreDeploy != nil {
+		ctr.ServiceSpec.PreDeploy.Env = nil
+	}
 
 	// Docker returns Mounts in a non-deterministic order so sort them.
 	slices.SortFunc(ctr.Mounts, func(a, b container.MountPoint) int {
@@ -267,6 +274,13 @@ func (s *Store) SubscribeContainers(ctx context.Context) ([]ContainerRecord, <-c
 	changes := make(chan struct{})
 	go func() {
 		defer close(changes)
+		// Coalesce bursts of rapid-fire row-level change events (e.g. from frequent health-check updates
+		// across many services) into a single signal per debounce window, instead of forwarding one signal
+		// per event. Every subscriber of this channel (e.g. the Caddy and DNS reconcilers) otherwise reruns
+		// its full reconciliation on every single event, which can burn significant CPU across the cluster
+		// when there's a lot of container churn.
+		var debouncer *time.Timer
+		var debouncerCh <-chan time.Time
 		for {
 			select {
 			case <-ctx.Done():
@@ -279,8 +293,21 @@ func (s *Store) SubscribeContainers(ctx context.Context) ([]ContainerRecord, <-c
 					}
 					return
 				}
-				// Just signal that there is a change in the containers list.
-				changes <- struct{}{}
+				if debouncerCh == nil {
+					if debouncer == nil {
+						debouncer = time.NewTimer(containerChangesDebounceInterval)
+					} else {
+						debouncer.Reset(containerChangesDebounceInterval)
+					}
+					debouncerCh = debouncer.C
+				}
+			case <-debouncerCh:
+				select {
+				case changes <- struct{}{}:
+				case <-ctx.Done():
+					return
+				}
+				debouncerCh = nil
 			}
 		}
 	}()

internal/machine/store/container_test.go

@@ -0,0 +1,67 @@
+package store
+
+import (
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestNormaliseContainerForStore_StripsEnv pins the behaviour that all fields carrying environment variables
+// are cleared before a container record is written to the replicated store, regardless of which part of the
+// spec they live in. See https://github.com/psviderski/uncloud/issues/422.
+func TestNormaliseContainerForStore_StripsEnv(t *testing.T) {
+	t.Parallel()
+
+	newContainer := func(preDeploy *api.PreDeployHook) api.ServiceContainer {
+		return api.ServiceContainer{
+			Container: api.Container{
+				InspectResponse: container.InspectResponse{
+					Config: &container.Config{
+						Env: []string{"CONTROL_TOKEN=control-value-aaa"},
+					},
+				},
+			},
+			ServiceSpec: api.ServiceSpec{
+				Container: api.ContainerSpec{
+					Env: api.EnvVars{"CONTROL_TOKEN": "control-value-aaa"},
+				},
+				PreDeploy: preDeploy,
+			},
+		}
+	}
+
+	t.Run("service with pre-deploy hook", func(t *testing.T) {
+		t.Parallel()
+
+		ctr := newContainer(&api.PreDeployHook{
+			Command: []string{"/bin/true"},
+			Env:     api.EnvVars{"HOOK_TOKEN": "hook-value-bbb"},
+		})
+
+		normaliseContainerForStore(&ctr)
+
+		assert.Nil(t, ctr.Config.Env, "Config.Env should be stripped")
+		assert.Nil(t, ctr.ServiceSpec.Container.Env, "ServiceSpec.Container.Env should be stripped")
+		require.NotNil(t, ctr.ServiceSpec.PreDeploy, "PreDeploy itself should be left in place")
+		assert.Nil(t, ctr.ServiceSpec.PreDeploy.Env, "ServiceSpec.PreDeploy.Env should be stripped")
+		// Non-env fields of the hook must survive the normalisation.
+		assert.Equal(t, []string{"/bin/true"}, ctr.ServiceSpec.PreDeploy.Command)
+	})
+
+	t.Run("service without pre-deploy hook", func(t *testing.T) {
+		t.Parallel()
+
+		ctr := newContainer(nil)
+
+		require.NotPanics(t, func() {
+			normaliseContainerForStore(&ctr)
+		})
+
+		assert.Nil(t, ctr.Config.Env)
+		assert.Nil(t, ctr.ServiceSpec.Container.Env)
+		assert.Nil(t, ctr.ServiceSpec.PreDeploy)
+	})
+}

mise.lock

@@ -1,36 +1,43 @@
 # @generated - this file is auto-generated by `mise lock` https://mise.jdx.dev/dev-tools/mise-lock.html
 
 [[tools."aqua:vektra/mockery"]]
-version = "3.5.3"
+version = "3.8.1"
 backend = "aqua:vektra/mockery"
 
 [tools."aqua:vektra/mockery"."platforms.linux-arm64"]
-checksum = "sha256:ec02ca72b7ad1ff5a1de5503a27d3b1b3f230d722eb35c5b92e43e295a8e1290"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Linux_arm64.tar.gz"
+checksum = "sha256:d1f481f3a6e9868dd3ad65da265238ee109ab3a5dd7508958d5811953696f452"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Linux_arm64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915757"
 
 [tools."aqua:vektra/mockery"."platforms.linux-arm64-musl"]
-checksum = "sha256:ec02ca72b7ad1ff5a1de5503a27d3b1b3f230d722eb35c5b92e43e295a8e1290"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Linux_arm64.tar.gz"
+checksum = "sha256:d1f481f3a6e9868dd3ad65da265238ee109ab3a5dd7508958d5811953696f452"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Linux_arm64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915757"
 
 [tools."aqua:vektra/mockery"."platforms.linux-x64"]
-checksum = "sha256:ebce416b0175338525246c376885a1579ca6cd4d4015140ba0c70e6b5339a39c"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Linux_x86_64.tar.gz"
+checksum = "sha256:953c7d38b58dd6398b1c595f27caf8bbf27e9b733b49c404d2e124e86d2ae72a"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Linux_x86_64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915756"
 
 [tools."aqua:vektra/mockery"."platforms.linux-x64-musl"]
-checksum = "sha256:ebce416b0175338525246c376885a1579ca6cd4d4015140ba0c70e6b5339a39c"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Linux_x86_64.tar.gz"
+checksum = "sha256:953c7d38b58dd6398b1c595f27caf8bbf27e9b733b49c404d2e124e86d2ae72a"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Linux_x86_64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915756"
 
 [tools."aqua:vektra/mockery"."platforms.macos-arm64"]
-checksum = "sha256:a3a94b14c7414e148f2252199ffc4a0108d311358f3d336cbe05bb73cb203704"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Darwin_arm64.tar.gz"
+checksum = "sha256:303a794484243906b37da0c37056f974bf94278c2e10d7b1372e6bd53a2d5034"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Darwin_arm64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915806"
 
 [tools."aqua:vektra/mockery"."platforms.macos-x64"]
-checksum = "sha256:8bd2b3e4722e476400395920fb5bbd68be4ea22ec12cddf1e424c40610fda863"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Darwin_x86_64.tar.gz"
+checksum = "sha256:6882331344e946268b304dce042b01371a0213a896536413be8abe49d532ca6f"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Darwin_x86_64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915746"
 
 [tools."aqua:vektra/mockery"."platforms.windows-x64"]
-checksum = "sha256:3b4264066db615a56a8b59abda992f6db82b8cd7666937ed9dd8f4a4bfc424a0"
-url = "https://github.com/vektra/mockery/releases/download/v3.5.3/mockery_3.5.3_Windows_x86_64.tar.gz"
+checksum = "sha256:7a7458756e9d7bce5f9936b7a63394000f3affa751b9740e65350c4bdcb627dc"
+url = "https://github.com/vektra/mockery/releases/download/v3.8.1/mockery_3.8.1_Windows_x86_64.tar.gz"
+url_api = "https://api.github.com/repos/vektra/mockery/releases/assets/632915759"
 
 [[tools.go]]
 version = "1.26.1"
@@ -71,32 +78,44 @@ backend = "aqua:golangci/golangci-lint"
 [tools.golangci-lint."platforms.linux-arm64"]
 checksum = "sha256:c3c83c7ed003e881991af21361c487151abd37969927ed9062cd0b48f3cd1d8d"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-linux-arm64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099658"
+provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.linux-arm64-musl"]
 checksum = "sha256:c3c83c7ed003e881991af21361c487151abd37969927ed9062cd0b48f3cd1d8d"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-linux-arm64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099658"
+provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.linux-x64"]
 checksum = "sha256:852741ea769f4543ab4b6cf86b313cb7173f2e8d2d39f00eb0cd5ef5d4d5eda9"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-linux-amd64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099674"
 provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.linux-x64-musl"]
 checksum = "sha256:852741ea769f4543ab4b6cf86b313cb7173f2e8d2d39f00eb0cd5ef5d4d5eda9"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-linux-amd64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099674"
+provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.macos-arm64"]
 checksum = "sha256:11e9549b955bae4bfb71aa3b5c2762e446e2d3f975fe963d6d767b8f745d9344"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-darwin-arm64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099671"
 provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.macos-x64"]
 checksum = "sha256:1dbbbe745eadba9cf012d49bd0b8c9290cbcb50dbf2cf9611f75a715b46ae3af"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-darwin-amd64.tar.gz"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099653"
+provenance = "github-attestations"
 
 [tools.golangci-lint."platforms.windows-x64"]
 checksum = "sha256:60e50178253d276d1c808ef00845eddcabca63b7d69d86a6dbaa41e88bc843f3"
 url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.2/golangci-lint-2.11.2-windows-amd64.zip"
+url_api = "https://api.github.com/repos/golangci/golangci-lint/releases/assets/369099651"
+provenance = "github-attestations"
 
 [[tools.goreleaser]]
 version = "2.15.2"
@@ -105,37 +124,44 @@ backend = "aqua:goreleaser/goreleaser"
 [tools.goreleaser."platforms.linux-arm64"]
 checksum = "sha256:5db66761a98f6693161e49e1a95d28d2673a892ba60cb4a5e16736cafd41c4c9"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Linux_arm64.tar.gz"
-provenance = "cosign"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702105"
+provenance = "github-attestations"
 
 [tools.goreleaser."platforms.linux-arm64-musl"]
 checksum = "sha256:5db66761a98f6693161e49e1a95d28d2673a892ba60cb4a5e16736cafd41c4c9"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Linux_arm64.tar.gz"
-provenance = "cosign"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702105"
+provenance = "github-attestations"
 
 [tools.goreleaser."platforms.linux-x64"]
 checksum = "sha256:0ebdbf0353aba566b969dde746cc4e4806f96c27aa2f3971b229a9df7611fedc"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Linux_x86_64.tar.gz"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702106"
 provenance = "github-attestations"
 
 [tools.goreleaser."platforms.linux-x64-musl"]
 checksum = "sha256:0ebdbf0353aba566b969dde746cc4e4806f96c27aa2f3971b229a9df7611fedc"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Linux_x86_64.tar.gz"
-provenance = "cosign"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702106"
+provenance = "github-attestations"
 
 [tools.goreleaser."platforms.macos-arm64"]
 checksum = "sha256:0e6bd67688ac949780bf1166813a91f89856898ef4c40d7d46c2c74ebaa4b9ee"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Darwin_all.tar.gz"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702148"
 provenance = "github-attestations"
 
 [tools.goreleaser."platforms.macos-x64"]
 checksum = "sha256:0e6bd67688ac949780bf1166813a91f89856898ef4c40d7d46c2c74ebaa4b9ee"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Darwin_all.tar.gz"
-provenance = "cosign"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702148"
+provenance = "github-attestations"
 
 [tools.goreleaser."platforms.windows-x64"]
 checksum = "sha256:7459832946dbe122c144f8d7f87484d8572ca005b779310aa6bb03346e8de17a"
 url = "https://github.com/goreleaser/goreleaser/releases/download/v2.15.2/goreleaser_Windows_x86_64.zip"
-provenance = "cosign"
+url_api = "https://api.github.com/repos/goreleaser/goreleaser/releases/assets/385702135"
+provenance = "github-attestations"
 
 [[tools.node]]
 version = "22.20.0"
@@ -146,16 +172,17 @@ checksum = "sha256:4181609e03dcb9880e7e5bf956061ecc0503c77a480c6631d868cb1f65a2c
 url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0-linux-arm64.tar.gz"
 
 [tools.node."platforms.linux-arm64-musl"]
-checksum = "sha256:4181609e03dcb9880e7e5bf956061ecc0503c77a480c6631d868cb1f65a2c7dd"
-url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0-linux-arm64.tar.gz"
+checksum = "sha256:79a087c1e23fc97d173ad6676df5b1a4e1a02daf1acca05ee25662747c5d492c"
+url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0.tar.gz"
+install = "source"
 
 [tools.node."platforms.linux-x64"]
 checksum = "sha256:eeaccb0378b79406f2208e8b37a62479c70595e20be6b659125eb77dd1ab2a29"
 url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0-linux-x64.tar.gz"
 
 [tools.node."platforms.linux-x64-musl"]
-checksum = "sha256:eeaccb0378b79406f2208e8b37a62479c70595e20be6b659125eb77dd1ab2a29"
-url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0-linux-x64.tar.gz"
+checksum = "sha256:461aafed5eb607357479c6de5df978515d74daf7e0169e972e61c8390c035f1a"
+url = "https://unofficial-builds.nodejs.org/download/release/v22.20.0/node-v22.20.0-linux-x64-musl.tar.gz"
 
 [tools.node."platforms.macos-arm64"]
 checksum = "sha256:cc04a76a09f79290194c0646f48fec40354d88969bec467789a5d55dd097f949"
@@ -173,9 +200,34 @@ url = "https://nodejs.org/dist/v22.20.0/node-v22.20.0-win-x64.zip"
 version = "27.3"
 backend = "aqua:protocolbuffers/protobuf/protoc"
 
+[tools.protoc."platforms.linux-arm64"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-linux-aarch_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006374"
+
+[tools.protoc."platforms.linux-arm64-musl"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-linux-aarch_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006374"
+
 [tools.protoc."platforms.linux-x64"]
 checksum = "blake3:8c2938accf7061164ea3968529a47ced1db4824ca553857f6903c68ae4347739"
 url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-linux-x86_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006380"
+
+[tools.protoc."platforms.linux-x64-musl"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-linux-x86_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006380"
+
+[tools.protoc."platforms.macos-arm64"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-osx-aarch_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006383"
+
+[tools.protoc."platforms.macos-x64"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-osx-x86_64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006381"
+
+[tools.protoc."platforms.windows-x64"]
+url = "https://github.com/protocolbuffers/protobuf/releases/download/v27.3/protoc-27.3-win64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf/releases/assets/183006385"
 
 [[tools.protoc-gen-go]]
 version = "1.34.2"
@@ -183,30 +235,62 @@ backend = "aqua:protocolbuffers/protobuf-go/protoc-gen-go"
 
 [tools.protoc-gen-go."platforms.linux-arm64"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.linux.arm64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109690"
 
 [tools.protoc-gen-go."platforms.linux-arm64-musl"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.linux.arm64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109690"
 
 [tools.protoc-gen-go."platforms.linux-x64"]
 checksum = "blake3:f765e393e0f0b25070c13848d88f5e6303e66090132afa1b753e981507a4174e"
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.linux.amd64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109692"
 
 [tools.protoc-gen-go."platforms.linux-x64-musl"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.linux.amd64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109692"
 
 [tools.protoc-gen-go."platforms.macos-arm64"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.darwin.arm64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109697"
 
 [tools.protoc-gen-go."platforms.macos-x64"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.darwin.amd64.tar.gz"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109698"
 
 [tools.protoc-gen-go."platforms.windows-x64"]
 url = "https://github.com/protocolbuffers/protobuf-go/releases/download/v1.34.2/protoc-gen-go.v1.34.2.windows.amd64.zip"
+url_api = "https://api.github.com/repos/protocolbuffers/protobuf-go/releases/assets/173109685"
 
 [[tools.protoc-gen-go-grpc]]
 version = "1.5.1"
 backend = "aqua:grpc/grpc-go/protoc-gen-go-grpc"
 
+[tools.protoc-gen-go-grpc."platforms.linux-arm64"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.linux.arm64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572238"
+
+[tools.protoc-gen-go-grpc."platforms.linux-arm64-musl"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.linux.arm64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572238"
+
 [tools.protoc-gen-go-grpc."platforms.linux-x64"]
 checksum = "blake3:197739132f0408f017d0c83773ac1ace85c0572473fdd658ec6e4ccbf20e0552"
 url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.linux.amd64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572229"
+
+[tools.protoc-gen-go-grpc."platforms.linux-x64-musl"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.linux.amd64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572229"
+
+[tools.protoc-gen-go-grpc."platforms.macos-arm64"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.darwin.arm64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572243"
+
+[tools.protoc-gen-go-grpc."platforms.macos-x64"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.darwin.amd64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572208"
+
+[tools.protoc-gen-go-grpc."platforms.windows-x64"]
+url = "https://github.com/grpc/grpc-go/releases/download/cmd/protoc-gen-go-grpc/v1.5.1/protoc-gen-go-grpc.v1.5.1.windows.amd64.tar.gz"
+url_api = "https://api.github.com/repos/grpc/grpc-go/releases/assets/182572216"

mise.toml

@@ -2,7 +2,7 @@
 experimental = true
 
 [tools]
-"aqua:vektra/mockery" = "3.5.3"
+"aqua:vektra/mockery" = "3.8.1"
 go = "1.26.1"
 golangci-lint = "2.11.2"
 goreleaser = "2.15.2"
@@ -40,6 +40,10 @@ outputs = ["uncloudd-linux-arm64"]
 description = "Update mise.lock with hashes for all platforms"
 run = "mise lock --platform linux-x64,macos-arm64"
 
+[tasks.mocks]
+description = "Regenerate Go test mocks"
+run = "mockery"
+
 [tasks.proto]
 description = "Regenerate gRPC API code from .proto files"
 run = """

pkg/api/client.go

@@ -42,6 +42,8 @@ type DNSClient interface {
 type ImageClient interface {
 	InspectImage(ctx context.Context, id string) ([]MachineImage, error)
 	InspectRemoteImage(ctx context.Context, id string) ([]MachineRemoteImage, error)
+	ListImages(ctx context.Context, filter ImageFilter) ([]MachineImages, error)
+	RemoveImage(ctx context.Context, image string, opts RemoveImageOptions) ([]MachineImageRemoval, error)
 }
 
 type LogsClient interface {

pkg/api/image.go

@@ -1,8 +1,6 @@
 package api
 
 import (
-	"errors"
-
 	"github.com/distribution/reference"
 	"github.com/docker/docker/api/types/image"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -24,14 +22,6 @@ type MachineImages struct {
 	ContainerdStore bool
 }
 
-// Error returns a non-nil error if listing images on this machine failed.
-func (mi MachineImages) Error() error {
-	if mi.Metadata != nil && mi.Metadata.Error != "" {
-		return errors.New(mi.Metadata.Error)
-	}
-	return nil
-}
-
 // ImageFilter defines criteria to filter images in ListImages.
 type ImageFilter struct {
 	// Machines filters images to those present on the specified machines (names or IDs).
@@ -42,6 +32,18 @@ type ImageFilter struct {
 	Name string
 }
 
+// RemoveImageOptions defines options for removing an image from cluster machines.
+type RemoveImageOptions struct {
+	// Machines selects machines to remove the image from (names or IDs). If empty, all cluster machines are selected.
+	Machines []string
+}
+
+// MachineImageRemoval contains the outcome of removing an image on one machine.
+type MachineImageRemoval struct {
+	Metadata *pb.Metadata
+	Results  []image.DeleteResponse
+}
+
 // MachineRemoteImage represents an image in a remote registry fetched by a particular machine.
 type MachineRemoteImage struct {
 	Metadata *pb.Metadata

pkg/api/machine.go

@@ -1,12 +1,33 @@
 package api
 
 import (
+	"fmt"
 	"net/netip"
 	"strings"
 
 	"github.com/psviderski/uncloud/api/pb"
 )
 
+// ValidateMachineName checks that a machine name is a lowercase DNS label and doesn't conflict with
+// internal DNS query modes or machine IDs.
+func ValidateMachineName(name string) error {
+	if !DNSLabelRegex.MatchString(name) {
+		return fmt.Errorf("invalid machine name %q: must be 1-63 characters, lowercase letters, numbers, "+
+			"and hyphens only, starting and ending with a letter or number", name)
+	}
+
+	switch name {
+	case "rr", "nearest":
+		return fmt.Errorf("invalid machine name %q: reserved for internal DNS query modes", name)
+	}
+	if IDRegex.MatchString(name) {
+		return fmt.Errorf(
+			"invalid machine name %q: must not match the machine ID format (32 hexadecimal characters)", name)
+	}
+
+	return nil
+}
+
 // MachineFilter defines criteria to filter machines in ListMachines.
 type MachineFilter struct {
 	// Available filters machines that are not DOWN.

pkg/api/machine_test.go

@@ -3,6 +3,7 @@ package api
 import (
 	"encoding/json"
 	"net/netip"
+	"strings"
 	"testing"
 
 	"github.com/psviderski/uncloud/api/pb"
@@ -10,6 +11,52 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestValidateMachineName(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		input   string
+		wantErr string
+	}{
+		{name: "single letter", input: "a"},
+		{name: "single digit", input: "1"},
+		{name: "two characters", input: "a1"},
+		{name: "generated name", input: "machine-ab12"},
+		{name: "hyphens and digits", input: "1-worker-2"},
+		{name: "maximum length", input: strings.Repeat("a", 63)},
+		{name: "maximum length with hyphens", input: "a" + strings.Repeat("-", 61) + "1"},
+		{name: "machine namespace label", input: "m"},
+		{name: "mode prefix", input: "nearest-worker"},
+		{name: "short hexadecimal name", input: strings.Repeat("a", 31)},
+		{name: "long hexadecimal name", input: strings.Repeat("a", 33)},
+		{name: "non-hexadecimal 32 characters", input: strings.Repeat("g", 32)},
+		{name: "empty", wantErr: "must be 1-63 characters"},
+		{name: "too long", input: strings.Repeat("a", 64), wantErr: "must be 1-63 characters"},
+		{name: "uppercase", input: "VPS1", wantErr: "lowercase letters"},
+		{name: "leading hyphen", input: "-worker", wantErr: "starting and ending"},
+		{name: "trailing hyphen", input: "worker-", wantErr: "starting and ending"},
+		{name: "underscore", input: "worker_1", wantErr: "hyphens only"},
+		{name: "dot", input: "worker.example", wantErr: "hyphens only"},
+		{name: "space", input: "worker 1", wantErr: "hyphens only"},
+		{name: "leading whitespace", input: " worker", wantErr: "hyphens only"},
+		{name: "trailing whitespace", input: "worker\t", wantErr: "hyphens only"},
+		{name: "non-ASCII", input: "wörker", wantErr: "lowercase letters"},
+		{name: "round-robin mode", input: "rr", wantErr: "reserved for internal DNS query modes"},
+		{name: "nearest mode", input: "nearest", wantErr: "reserved for internal DNS query modes"},
+		{name: "machine ID", input: "c337f00600de51ef4375c9a9a267dba5", wantErr: "machine ID format"},
+	}
+
+	for _, tt := range tests {
+		err := ValidateMachineName(tt.input)
+		if tt.wantErr == "" {
+			require.NoError(t, err)
+		} else {
+			require.ErrorContains(t, err, tt.wantErr)
+		}
+	}
+}
+
 func TestMachineMembersList_Info(t *testing.T) {
 	t.Parallel()
 

pkg/api/service.go

@@ -41,12 +41,30 @@ const (
 )
 
 var (
-	serviceIDRegexp = regexp.MustCompile("^[0-9a-f]{32}$")
-	dnsLabelRegexp  = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`)
+	IDRegex       = regexp.MustCompile("^[0-9a-f]{32}$")
+	DNSLabelRegex = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$`)
 )
 
 func ValidateServiceID(id string) bool {
-	return serviceIDRegexp.MatchString(id)
+	return IDRegex.MatchString(id)
+}
+
+// ValidateServiceName checks that a service name is a lowercase DNS label and doesn't conflict with
+// the machine DNS namespace or service IDs.
+func ValidateServiceName(name string) error {
+	if !DNSLabelRegex.MatchString(name) {
+		return fmt.Errorf("invalid service name %q: must be 1-63 characters, lowercase letters, numbers, "+
+			"and hyphens only, starting and ending with a letter or number", name)
+	}
+	if name == "m" {
+		return fmt.Errorf("invalid service name %q: reserved for the machine DNS namespace", name)
+	}
+	if IDRegex.MatchString(name) {
+		return fmt.Errorf(
+			"invalid service name %q: must not match the service ID format (32 hexadecimal characters)", name)
+	}
+
+	return nil
 }
 
 // ServiceSpec defines the desired state of a service.
@@ -147,12 +165,8 @@ func (s *ServiceSpec) Validate() error {
 	}
 
 	if s.Name != "" {
-		if len(s.Name) > 63 {
-			return fmt.Errorf("service name too long (max 63 characters): %q", s.Name)
-		}
-		if !dnsLabelRegexp.MatchString(s.Name) {
-			return fmt.Errorf("invalid service name: %q. must be 1-63 characters, lowercase letters, numbers, "+
-				"and dashes only; must start and end with a letter or number", s.Name)
+		if err := ValidateServiceName(s.Name); err != nil {
+			return err
 		}
 	}
 

pkg/api/service_test.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/docker/docker/api/types/container"
@@ -9,6 +10,59 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestServiceSpec_Validate_Name(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		input   string
+		wantErr string
+	}{
+		{name: "empty allows generated name"},
+		{name: "single character", input: "a"},
+		{name: "single digit", input: "1"},
+		{name: "two characters", input: "a1"},
+		{name: "hyphens and digits", input: "1-web-2"},
+		{name: "consecutive hyphens", input: "web--1"},
+		{name: "maximum length", input: strings.Repeat("a", 63)},
+		{name: "maximum length with hyphens", input: "a" + strings.Repeat("-", 61) + "1"},
+		{name: "round-robin mode is a valid service name", input: "rr"},
+		{name: "nearest mode is a valid service name", input: "nearest"},
+		{name: "machine namespace prefix", input: "m-service"},
+		{name: "short hexadecimal name", input: strings.Repeat("a", 31)},
+		{name: "long hexadecimal name", input: strings.Repeat("a", 33)},
+		{name: "non-hexadecimal 32 characters", input: strings.Repeat("g", 32)},
+		{name: "too long", input: strings.Repeat("a", 64), wantErr: "must be 1-63 characters"},
+		{name: "uppercase", input: "WEB1", wantErr: "lowercase letters"},
+		{name: "leading hyphen", input: "-web", wantErr: "starting and ending"},
+		{name: "trailing hyphen", input: "web-", wantErr: "starting and ending"},
+		{name: "hyphen only", input: "-", wantErr: "starting and ending"},
+		{name: "underscore", input: "web_1", wantErr: "hyphens only"},
+		{name: "dot", input: "web.example", wantErr: "hyphens only"},
+		{name: "slash", input: "web/api", wantErr: "hyphens only"},
+		{name: "space", input: "web 1", wantErr: "hyphens only"},
+		{name: "leading whitespace", input: " web", wantErr: "hyphens only"},
+		{name: "trailing whitespace", input: "web ", wantErr: "hyphens only"},
+		{name: "tab", input: "web\t1", wantErr: "hyphens only"},
+		{name: "newline", input: "web\n", wantErr: "hyphens only"},
+		{name: "non-ASCII", input: "wéb", wantErr: "lowercase letters"},
+		{name: "machine DNS namespace", input: "m", wantErr: "reserved for the machine DNS namespace"},
+		{name: "service ID", input: "c337f00600de51ef4375c9a9a267dba5", wantErr: "service ID format"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			spec := ServiceSpec{Name: tt.input, Container: ContainerSpec{Image: "nginx:latest"}}
+			err := spec.Validate()
+			if tt.wantErr != "" {
+				require.ErrorContains(t, err, tt.wantErr)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
 func TestServiceSpec_Validate_CaddyAndPorts(t *testing.T) {
 	tests := []struct {
 		name    string

pkg/client/client.go

@@ -86,6 +86,7 @@ func (cli *Client) progressOut() *streams.Out {
 // ProxyMachinesContext returns a new context that proxies gRPC requests to the specified machines.
 // If namesOrIDs is nil or empty, all machines are included.
 // This triggers One2Many proxying, which always injects metadata into the response.
+// TODO: validate namesOrIDs can't contain "*" as this is reserved as internal value and may break the caller logic.
 func ProxyMachinesContext(ctx context.Context, namesOrIDs []string) context.Context {
 	md := outgoingMetadataWithoutProxyTargets(ctx)
 	if len(namesOrIDs) == 0 {

pkg/client/compose/deploy.go

@@ -13,8 +13,8 @@ import (
 	"github.com/docker/docker/api/types/volume"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client/deploy"
-	"github.com/psviderski/uncloud/pkg/client/deploy/operation"
 	"github.com/psviderski/uncloud/pkg/client/deploy/scheduler"
+	"github.com/psviderski/uncloud/pkg/client/operation"
 )
 
 type Client interface {

pkg/client/compose/plan.go

@@ -10,7 +10,7 @@ import (
 	"github.com/psviderski/uncloud/internal/cli/tui"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client/deploy"
-	"github.com/psviderski/uncloud/pkg/client/deploy/operation"
+	"github.com/psviderski/uncloud/pkg/client/operation"
 )
 
 // Plan holds the compose-level deployment plan with volume and service operations.

pkg/client/compose/secret.go

@@ -89,40 +89,53 @@ func ResolveSecrets(ctx context.Context, project *types.Project) error {
 		return v, nil
 	}
 
-	// Resolve only secret references set as values for environment variables in enabled services.
 	for _, service := range project.Services {
-		for k, v := range service.Environment {
-			if v == nil {
-				continue
-			}
-			secretName, ok := secretRefName(*v)
-			if !ok {
-				continue
-			}
-			value, err := resolve(secretName)
-			if err != nil {
+		// Resolve secret references set as values for environment variables in the service.
+		if err := resolveEnvSecrets(service.Environment, resolve); err != nil {
+			return err
+		}
+
+		// Resolve secret references set as values for the pre-deploy hook's environment variables, if any.
+		if hook, ok := service.Extensions[PreDeployHookExtensionKey].(PreDeployHook); ok {
+			if err := resolveEnvSecrets(hook.Environment, resolve); err != nil {
 				return err
 			}
-			service.Environment[k] = &value
 		}
 	}
 
 	return nil
 }
 
-// HasCommandSecretRefs reports whether any service environment references a secret that is resolved by running
-// a command.
+// resolveEnvSecrets resolves 'secret://name' references found as values in env to actual secret values using
+// resolve, setting each referenced variable to the secret's value in place. Variables that are not secret
+// references are left untouched.
+func resolveEnvSecrets(env types.MappingWithEquals, resolve func(string) (string, error)) error {
+	for k, v := range env {
+		if v == nil {
+			continue
+		}
+		secretName, ok := secretRefName(*v)
+		if !ok {
+			continue
+		}
+		value, err := resolve(secretName)
+		if err != nil {
+			return err
+		}
+		env[k] = &value
+	}
+	return nil
+}
+
+// HasCommandSecretRefs reports whether any service environment, including a pre-deploy hook's environment,
+// references a secret that is resolved by running a command.
 func HasCommandSecretRefs(project *types.Project) bool {
 	for _, service := range project.Services {
-		for _, v := range service.Environment {
-			if v == nil {
-				continue
-			}
-			name, ok := secretRefName(*v)
-			if !ok {
-				continue
-			}
-			if secret, ok := project.Secrets[name]; ok && secret.Driver == secretExecDriver {
+		if envHasCommandSecretRef(service.Environment, project) {
+			return true
+		}
+		if hook, ok := service.Extensions[PreDeployHookExtensionKey].(PreDeployHook); ok {
+			if envHasCommandSecretRef(hook.Environment, project) {
 				return true
 			}
 		}
@@ -130,6 +143,24 @@ func HasCommandSecretRefs(project *types.Project) bool {
 	return false
 }
 
+// envHasCommandSecretRef reports whether env contains a 'secret://name' reference to a secret resolved
+// by running a command.
+func envHasCommandSecretRef(env types.MappingWithEquals, project *types.Project) bool {
+	for _, v := range env {
+		if v == nil {
+			continue
+		}
+		name, ok := secretRefName(*v)
+		if !ok {
+			continue
+		}
+		if secret, ok := project.Secrets[name]; ok && secret.Driver == secretExecDriver {
+			return true
+		}
+	}
+	return false
+}
+
 // secretValue resolves a secret to its value depending on its source: an 'exec' driver command,
 // an environment variable, or a file. A single trailing newline ('\n' or '\r\n') is stripped from the command
 // output as command-line tools commonly append one. All other whitespace is kept. Environment and file values are

pkg/client/compose/secret_test.go

@@ -328,6 +328,74 @@ secrets:
 	assert.Equal(t, "run\n", string(runs), "command should run exactly once across services and repeated resolutions")
 }
 
+// TestResolveSecrets_PreDeployHook pins the behaviour that 'secret://name' references in the pre-deploy hook's
+// environment are resolved just like references in the service's own environment.
+// See https://github.com/psviderski/uncloud/issues/422.
+func TestResolveSecrets_PreDeployHook(t *testing.T) {
+	t.Parallel()
+
+	content := `
+services:
+  foo:
+    image: foo
+    environment:
+      CONTROL_TOKEN: secret://control_token
+    x-pre_deploy:
+      command: ["./migrate.sh"]
+      environment:
+        HOOK_TOKEN: secret://hook_token
+        PLAIN: hello
+secrets:
+  control_token:
+    x-command: printf control-value
+  hook_token:
+    x-command: printf hook-value
+`
+	project := loadProject(t, content)
+	require.NoError(t, ResolveSecrets(context.Background(), project))
+
+	assert.Equal(t, env(map[string]string{"CONTROL_TOKEN": "control-value"}), project.Services["foo"].Environment)
+
+	hook, ok := project.Services["foo"].Extensions[PreDeployHookExtensionKey].(PreDeployHook)
+	require.True(t, ok, "x-pre_deploy extension not found")
+	assert.Equal(t, env(map[string]string{"HOOK_TOKEN": "hook-value", "PLAIN": "hello"}), hook.Environment)
+}
+
+// TestResolveSecrets_PreDeployHook_ResolvedOnce verifies that a secret referenced from both the service's
+// environment and its pre-deploy hook's environment is only resolved once.
+func TestResolveSecrets_PreDeployHook_ResolvedOnce(t *testing.T) {
+	t.Parallel()
+
+	counter := filepath.Join(t.TempDir(), "runs")
+	content := fmt.Sprintf(`
+services:
+  foo:
+    image: foo
+    environment:
+      TOKEN: secret://token
+    x-pre_deploy:
+      command: ["./migrate.sh"]
+      environment:
+        TOKEN: secret://token
+secrets:
+  token:
+    x-command: "sh -c 'echo run >> %s; printf abc'"
+`, counter)
+
+	project := loadProject(t, content)
+	require.NoError(t, ResolveSecrets(context.Background(), project))
+
+	assert.Equal(t, "abc", *project.Services["foo"].Environment["TOKEN"])
+	hook, ok := project.Services["foo"].Extensions[PreDeployHookExtensionKey].(PreDeployHook)
+	require.True(t, ok, "x-pre_deploy extension not found")
+	assert.Equal(t, "abc", *hook.Environment["TOKEN"])
+
+	runs, err := os.ReadFile(counter)
+	require.NoError(t, err)
+	assert.Equal(t, "run\n", string(runs),
+		"command should run exactly once across the service and hook environments")
+}
+
 func TestHasCommandSecretRefs(t *testing.T) {
 	t.Parallel()
 
@@ -387,6 +455,22 @@ services:
 `,
 			want: false,
 		},
+		{
+			name: "command secret referenced from pre-deploy hook",
+			content: `
+services:
+  foo:
+    image: foo
+    x-pre_deploy:
+      command: ["./migrate.sh"]
+      environment:
+        TOKEN: secret://token
+secrets:
+  token:
+    x-command: printf abc
+`,
+			want: true,
+		},
 	}
 
 	for _, tt := range tests {

pkg/client/deploy/deploy.go

@@ -12,8 +12,8 @@ import (
 	"github.com/distribution/reference"
 	"github.com/psviderski/uncloud/internal/cli/tui"
 	"github.com/psviderski/uncloud/pkg/api"
-	"github.com/psviderski/uncloud/pkg/client/deploy/operation"
 	"github.com/psviderski/uncloud/pkg/client/deploy/scheduler"
+	"github.com/psviderski/uncloud/pkg/client/operation"
 )
 
 type Client interface {

pkg/client/deploy/strategy.go

@@ -8,8 +8,8 @@ import (
 	"github.com/psviderski/uncloud/api/pb"
 	"github.com/psviderski/uncloud/internal/secret"
 	"github.com/psviderski/uncloud/pkg/api"
-	"github.com/psviderski/uncloud/pkg/client/deploy/operation"
 	"github.com/psviderski/uncloud/pkg/client/deploy/scheduler"
+	"github.com/psviderski/uncloud/pkg/client/operation"
 )
 
 // Strategy defines how a service should be deployed or updated. Different implementations can provide various

pkg/client/deploy/strategy_test.go

@@ -8,8 +8,8 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/psviderski/uncloud/api/pb"
 	"github.com/psviderski/uncloud/pkg/api"
-	"github.com/psviderski/uncloud/pkg/client/deploy/operation"
 	"github.com/psviderski/uncloud/pkg/client/deploy/scheduler"
+	"github.com/psviderski/uncloud/pkg/client/operation"
 	"github.com/stretchr/testify/assert"
 )
 

pkg/client/image.go

@@ -23,9 +23,11 @@ import (
 	"github.com/docker/docker/api/types/image"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/jsonmessage"
+	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/go-connections/nat"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/psviderski/uncloud/api/pb"
+	cliprogress "github.com/psviderski/uncloud/internal/cli/progress"
 	"github.com/psviderski/uncloud/internal/cli/tui"
 	"github.com/psviderski/uncloud/internal/docker"
 	"github.com/psviderski/uncloud/internal/machine/constants"
@@ -34,6 +36,8 @@ import (
 	"github.com/psviderski/uncloud/internal/secret"
 	"github.com/psviderski/uncloud/pkg/api"
 	netproxy "golang.org/x/net/proxy"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 // This is the container image used to run socat proxy containers.
@@ -61,7 +65,7 @@ func (cli *Client) ListImages(ctx context.Context, filter api.ImageFilter) ([]ap
 	// Broadcast the image list request to the specified machines or all machines if none specified.
 	listCtx := ProxyMachinesContext(ctx, filter.Machines)
 
-	opts := image.ListOptions{Manifests: true}
+	opts := image.ListOptions{All: true, Manifests: true}
 	if filter.Name != "" {
 		opts.Filters = filters.NewArgs(
 			filters.Arg("reference", filter.Name),
@@ -92,7 +96,7 @@ func (cli *Client) ListImages(ctx context.Context, filter api.ImageFilter) ([]ap
 			ContainerdStore: msg.ContainerdStore,
 		}
 
-		if msg.Metadata.Error == "" && len(msg.Images) > 0 {
+		if msg.Metadata.Err() == nil && len(msg.Images) > 0 {
 			if err = json.Unmarshal(msg.Images, &mi.Images); err != nil {
 				return nil, fmt.Errorf("unmarshal images: %w", err)
 			}
@@ -104,6 +108,81 @@ func (cli *Client) ListImages(ctx context.Context, filter api.ImageFilter) ([]ap
 	return machineImages, nil
 }
 
+// RemoveImage removes one exact image name or ID on the selected machines with protection of used references.
+// Results include the outcome of removing an image on each machine.
+//
+// An absent image on some machines is normal if removal succeeds on at least one machine. If every selected machine
+// reports NotFound, the returned error is api.ErrNotFound. Other machine failures are combined, with successful
+// results still returned.
+func (cli *Client) RemoveImage(
+	ctx context.Context, imageNameOrID string, opts api.RemoveImageOptions,
+) ([]api.MachineImageRemoval, error) {
+	if strings.TrimSpace(imageNameOrID) == "" {
+		return nil, errors.New("image name or ID is required")
+	}
+
+	resp, err := cli.Docker.GRPCClient.RemoveImage(ProxyMachinesContext(ctx, opts.Machines),
+		&pb.RemoveImageRequest{Id: imageNameOrID, PruneChildren: true})
+	if err != nil {
+		return nil, err
+	}
+
+	pw := progress.ContextWriter(ctx)
+	var results []api.MachineImageRemoval
+	var errs []error
+	notFoundCount := 0
+	for _, msg := range resp.Messages {
+		result := api.MachineImageRemoval{Metadata: msg.Metadata}
+		for _, removal := range msg.Results {
+			result.Results = append(result.Results, image.DeleteResponse{
+				Deleted:  removal.Deleted,
+				Untagged: removal.Untagged,
+			})
+		}
+		results = append(results, result)
+
+		eventID := cliprogress.ImageEventID(imageNameOrID, msg.Metadata.MachineName)
+		if err = result.Metadata.Err(); err != nil {
+			event := progress.NewEvent(eventID, progress.Done, "Not found")
+			if status.Code(err) == codes.NotFound {
+				notFoundCount++
+			} else {
+				errs = append(errs, fmt.Errorf("machine '%s': %w", msg.Metadata.MachineName, err))
+				event.Status = progress.Error
+				event.StatusText = status.Convert(err).Message()
+			}
+			pw.Event(event)
+			continue
+		}
+
+		// An unnamed image can return Deleted without Untagged. Use the requested name or ID as its row fallback.
+		deleted := false
+		for _, removal := range result.Results {
+			if removal.Untagged != "" {
+				eventID = cliprogress.ImageEventID(removal.Untagged, msg.Metadata.MachineName)
+				pw.Event(progress.NewEvent(eventID, progress.Done, "Untagged"))
+			}
+			if removal.Deleted != "" && !deleted {
+				// Docker reports the deletion of the selected image before any pruned parents or released layers.
+				// Only that first deletion updates the last untagged row marking the image as Removed.
+				deleted = true
+				pw.Event(progress.NewEvent(eventID, progress.Done,
+					fmt.Sprintf("Removed (%s)", stringid.TruncateID(removal.Deleted))))
+			}
+		}
+	}
+
+	if err = errors.Join(errs...); err != nil {
+		return results, err
+	}
+	if notFoundCount == len(results) {
+		return results, api.ErrNotFound
+	}
+
+	return results, nil
+}
+
+// TODO: move PushImageOptions to api package and include PushImage in ImageClient interface.
 type PushImageOptions struct {
 	// AllMachines pushes the image to all machines in the cluster. Takes precedence over Machines field.
 	AllMachines bool
@@ -215,7 +294,7 @@ func (cli *Client) pushImageToMachine(
 		Name:     "%invalid-name-to-only-check-store-type%",
 	})
 	if err == nil {
-		err = images[0].Error()
+		err = images[0].Metadata.Err()
 	}
 	if err != nil {
 		return fmt.Errorf("check Docker image store type on machine '%s': %w", machine.Name, err)

pkg/client/operation/container.go

@@ -173,6 +173,16 @@ type ReplaceContainerOperation struct {
 func (o *ReplaceContainerOperation) Execute(ctx context.Context, cli Client) error {
 	stopFirst := o.Order == api.UpdateOrderStopFirst
 
+	// Create the new container before stopping the old one. CreateContainer also pulls the image according to the
+	// service pull policy, so the stop-first downtime only includes stopping the old container and starting and
+	// monitoring the new one.
+	resp, err := cli.CreateContainer(ctx, o.ServiceID, o.Spec, o.MachineID)
+	if err != nil {
+		return fmt.Errorf("create new container: %w", err)
+	}
+	// Override event ID so StartContainer and WaitContainerHealthy update the same progress line as creation.
+	newCtx := cliprogress.WithEventID(ctx, cliprogress.NewContainerEventID(ctx, resp.Name, o.MachineName))
+
 	wasRunning := false
 	if stopFirst {
 		// Inspect the old container to remember its running state before stopping.
@@ -189,12 +199,6 @@ func (o *ReplaceContainerOperation) Execute(ctx context.Context, cli Client) err
 		}
 	}
 
-	resp, err := cli.CreateContainer(ctx, o.ServiceID, o.Spec, o.MachineID)
-	if err != nil {
-		return fmt.Errorf("create new container: %w", err)
-	}
-	// Override event ID so StartContainer and WaitContainerHealthy update the same progress line as creation.
-	newCtx := cliprogress.WithEventID(ctx, cliprogress.NewContainerEventID(ctx, resp.Name, o.MachineName))
 	if err = cli.StartContainer(newCtx, o.ServiceID, resp.ID); err != nil {
 		return fmt.Errorf("start new container: %w", err)
 	}

pkg/client/operation/container_test.go

@@ -0,0 +1,155 @@
+package operation
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/volume"
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestReplaceContainerOperation_StopFirstPreparation(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name      string
+		createErr error
+		wantErr   string
+		wantCalls []string
+	}{
+		{
+			name: "prepare before cutover",
+			wantCalls: []string{
+				"create:service",
+				"inspect:old",
+				"stop:old",
+				"start:new",
+				"wait:new",
+				"remove:old",
+			},
+		},
+		{
+			name:      "create failure does not interrupt old container",
+			createErr: errors.New("pull failed"),
+			wantErr:   "create new container: pull failed",
+			wantCalls: []string{"create:service"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			cli := &recordingClient{
+				oldContainerRunning: true,
+				createErr:           tt.createErr,
+			}
+			op := newStopFirstReplaceContainerOperation()
+
+			err := op.Execute(context.Background(), cli)
+			if tt.wantErr != "" {
+				require.ErrorContains(t, err, tt.wantErr)
+			} else {
+				require.NoError(t, err)
+			}
+			assert.Equal(t, tt.wantCalls, cli.calls)
+		})
+	}
+}
+
+func newStopFirstReplaceContainerOperation() *ReplaceContainerOperation {
+	return &ReplaceContainerOperation{
+		ServiceID:    "service-id",
+		Spec:         api.ServiceSpec{Name: "service"},
+		MachineID:    "machine-id",
+		MachineName:  "machine",
+		OldContainer: api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{ContainerJSONBase: &container.ContainerJSONBase{ID: "old"}}}},
+		Order:        api.UpdateOrderStopFirst,
+	}
+}
+
+type recordingClient struct {
+	calls               []string
+	oldContainerRunning bool
+	createErr           error
+}
+
+func (c *recordingClient) CreateContainer(
+	_ context.Context, _ string, spec api.ServiceSpec, _ string,
+) (api.CreateContainerResponse, error) {
+	c.calls = append(c.calls, "create:"+spec.Name)
+	return api.CreateContainerResponse{
+		CreateResponse: container.CreateResponse{ID: "new"},
+		Name:           "new",
+	}, c.createErr
+}
+
+func (c *recordingClient) InspectContainer(
+	_ context.Context, _, containerID string,
+) (api.MachineServiceContainer, error) {
+	c.calls = append(c.calls, "inspect:"+containerID)
+	return api.MachineServiceContainer{
+		Container: api.ServiceContainer{
+			Container: api.Container{InspectResponse: container.InspectResponse{
+				ContainerJSONBase: &container.ContainerJSONBase{
+					ID:    containerID,
+					State: &container.State{Running: c.oldContainerRunning},
+				},
+			}},
+		},
+	}, nil
+}
+
+func (c *recordingClient) StartContainer(_ context.Context, _, containerID string) error {
+	c.calls = append(c.calls, "start:"+containerID)
+	return nil
+}
+
+func (c *recordingClient) StopContainer(
+	_ context.Context, _, containerID string, _ container.StopOptions,
+) error {
+	c.calls = append(c.calls, "stop:"+containerID)
+	return nil
+}
+
+func (c *recordingClient) RemoveContainer(
+	_ context.Context, _, containerID string, _ container.RemoveOptions,
+) error {
+	c.calls = append(c.calls, "remove:"+containerID)
+	return nil
+}
+
+func (c *recordingClient) CreatePreDeployHookContainer(
+	context.Context, string, api.ServiceSpec, string,
+) (api.CreateContainerResponse, error) {
+	panic("unexpected CreatePreDeployHookContainer call")
+}
+
+func (c *recordingClient) ExecContainer(context.Context, string, string, api.ExecOptions) (int, error) {
+	panic("unexpected ExecContainer call")
+}
+
+func (c *recordingClient) WaitContainerHealthy(
+	_ context.Context, _, containerID string, _ api.WaitContainerHealthyOptions,
+) error {
+	c.calls = append(c.calls, "wait:"+containerID)
+	return nil
+}
+
+func (c *recordingClient) CreateVolume(
+	context.Context, string, volume.CreateOptions,
+) (api.MachineVolume, error) {
+	panic("unexpected CreateVolume call")
+}
+
+func (c *recordingClient) ListVolumes(context.Context, *api.VolumeFilter) ([]api.MachineVolume, error) {
+	panic("unexpected ListVolumes call")
+}
+
+func (c *recordingClient) RemoveVolume(context.Context, string, string, bool) error {
+	panic("unexpected RemoveVolume call")
+}

pkg/client/operation/operation.go

@@ -6,8 +6,8 @@ import (
 	"github.com/psviderski/uncloud/pkg/api"
 )
 
-// Operation represents a single atomic operation in a deployment process.
-// Operations can be composed to form complex deployment strategies.
+// Operation represents a single operation in the cluster that can be executed.
+// Operations can be composed to form complex workflows (plans).
 type Operation interface {
 	// Execute performs the operation using the provided client.
 	Execute(ctx context.Context, cli Client) error
@@ -16,7 +16,7 @@ type Operation interface {
 	String() string
 }
 
-// Client defines the interface required to execute deployment operations.
+// Client defines the interface required to execute cluster operations.
 type Client interface {
 	api.ContainerClient
 	api.VolumeClient

pkg/client/service.go

@@ -125,10 +125,10 @@ func (cli *Client) InspectService(ctx context.Context, nameOrID string) (api.Ser
 			continue
 		}
 
-		if mc.Metadata.Error != "" {
+		if err := mc.Metadata.Err(); err != nil {
 			// TODO: return failed machines in the response.
 			tui.PrintWarning(fmt.Sprintf("failed to list containers on machine '%s': %s",
-				mc.Metadata.MachineName, mc.Metadata.Error))
+				mc.Metadata.MachineName, err))
 			continue
 		}
 
@@ -333,10 +333,10 @@ func (cli *Client) ListServices(ctx context.Context) ([]api.Service, error) {
 	// Group containers by service ID using the broadcast results.
 	servicesByID := make(map[string]api.Service)
 	for _, mc := range machineContainers {
-		if mc.Metadata.Error != "" {
+		if err := mc.Metadata.Err(); err != nil {
 			// TODO: return failed machines in the response.
 			tui.PrintWarning(fmt.Sprintf("failed to list containers on machine '%s': %s",
-				mc.Metadata.MachineName, mc.Metadata.Error))
+				mc.Metadata.MachineName, err))
 			continue
 		}
 

pkg/client/volume.go

@@ -70,10 +70,10 @@ func (cli *Client) ListVolumes(ctx context.Context, filter *api.VolumeFilter) ([
 			continue
 		}
 
-		if mv.Metadata.Error != "" {
+		if err := mv.Metadata.Err(); err != nil {
 			// TODO: return failed machines in the response.
 			tui.PrintWarning(fmt.Sprintf("failed to list volumes on machine '%s': %s", mv.Metadata.MachineName,
-				mv.Metadata.Error))
+				err))
 			continue
 		}
 

test/e2e/service_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/go-units"
@@ -1265,10 +1266,10 @@ myapp.example.com {
 			"Containers should be on machines where the volume exists")
 	})
 
-	t.Run("list images for replicated service", func(t *testing.T) {
+	t.Run("image lifecycle for replicated service", func(t *testing.T) {
 		t.Parallel()
 
-		serviceName := "test-list-images-replicated"
+		serviceName := "test-image-lifecycle-replicated"
 		uniqueImage := "portainer/pause:3.9" // Unique image not used by other tests.
 		t.Cleanup(func() {
 			err := cli.RemoveService(ctx, serviceName)
@@ -1301,39 +1302,59 @@ myapp.example.com {
 		machinesWithContainers := serviceMachines(svc)
 		assert.Len(t, machinesWithContainers.ToSlice(), 2, "Containers should be on 2 different machines")
 
-		var machineWithoutContainer string
-		for _, m := range c.Machines {
-			if !machinesWithContainers.Contains(m.ID) {
-				machineWithoutContainer = m.ID
-				break
-			}
-		}
-		assert.NotEmpty(t, machineWithoutContainer, "Should have found a machine without container")
-
-		machineImages, err := cli.ListImages(ctx, api.ImageFilter{})
+		machineImages, err := cli.ListImages(ctx, api.ImageFilter{Name: uniqueImage})
 		require.NoError(t, err)
-		assert.Len(t, machineImages, 3, "Should get images from all 3 machines")
+		require.Len(t, machineImages, len(c.Machines))
 
 		for _, mi := range machineImages {
-			// Checking only DockerImages because the machines in ucind cluster don't use the containerd image store.
+			require.NoError(t, mi.Metadata.Err())
 			if !machinesWithContainers.Contains(mi.Metadata.MachineId) {
-				// This is the machine without service containers, it should not have the unique image.
-				for _, img := range mi.Images {
-					assert.NotContains(t, img.RepoTags, uniqueImage)
-				}
+				assert.Empty(t, mi.Images)
 				continue
 			}
 
-			// Check if the unique image is present on the machine where a service container is running.
-			hasImage := false
-			for _, img := range mi.Images {
-				if slices.Contains(img.RepoTags, uniqueImage) {
-					hasImage = true
-					break
-				}
+			require.Len(t, mi.Images, 1)
+			assert.Contains(t, mi.Images[0].RepoTags, uniqueImage)
+		}
+
+		// The used tag must remain on both machines while their service containers exist.
+		results, err := cli.RemoveImage(ctx, uniqueImage, api.RemoveImageOptions{Machines: spec.Placement.Machines})
+		require.Error(t, err)
+		assert.ErrorContains(t, err, "used by container")
+		require.Len(t, results, 2)
+		for _, result := range results {
+			assert.Equal(t, codes.FailedPrecondition, status.Code(result.Metadata.Err()))
+		}
+
+		require.NoError(t, cli.RemoveService(ctx, serviceName))
+
+		// Broadcast removal after deleting the containers. Absence on the third machine is not an overall error.
+		results, err = cli.RemoveImage(ctx, uniqueImage, api.RemoveImageOptions{})
+		require.NoError(t, err)
+		require.Len(t, results, len(c.Machines))
+		for _, result := range results {
+			if !machinesWithContainers.Contains(result.Metadata.MachineId) {
+				assert.Equal(t, codes.NotFound, status.Code(result.Metadata.Err()))
+				continue
 			}
-			assert.True(t, hasImage, "Machine %s with container should have image %s",
-				mi.Metadata.MachineId, uniqueImage)
+			require.NoError(t, result.Metadata.Err())
+			assert.Contains(t, result.Results, image.DeleteResponse{Untagged: uniqueImage})
+		}
+
+		results, err = cli.RemoveImage(ctx, uniqueImage, api.RemoveImageOptions{})
+		require.ErrorIs(t, err, api.ErrNotFound)
+		require.Len(t, results, len(c.Machines))
+		for _, result := range results {
+			assert.Equal(t, codes.NotFound, status.Code(result.Metadata.Err()))
+			assert.Empty(t, result.Results)
+		}
+
+		machineImages, err = cli.ListImages(ctx, api.ImageFilter{Name: uniqueImage})
+		require.NoError(t, err)
+		require.Len(t, machineImages, len(c.Machines))
+		for _, mi := range machineImages {
+			require.NoError(t, mi.Metadata.Err())
+			assert.Empty(t, mi.Images)
 		}
 	})
 

website/docs/3-concepts/2-ingress/4-cluster-storage-for-caddy.md

@@ -15,6 +15,15 @@ includes it and configure Caddy to use it.
 
 ## Enabling cluster storage
 
+:::warning Certificate migration
+
+Caddy doesn't migrate existing certificates automatically when you change its storage backend. **All certificates will
+need to be reissued unless you migrate your existing storage first.** Use the experimental
+[`caddy storage export` and `caddy storage import`](https://caddyserver.com/docs/command-line#caddy-storage)
+commands with the old and new configs to transfer the storage contents.
+
+:::
+
 :::info Requirements
 
 Cluster storage requires Uncloud **v0.21.0 or newer** for both the `uc` CLI and the daemon on every cluster machine.
@@ -37,7 +46,7 @@ Create a global Caddyfile, or add `storage uncloud` to the global options in you
 Deploy Caddy with the pre-built module image and your global config:
 
 ```shell
-uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.1 --caddyfile global.Caddyfile
+uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.3 --caddyfile global.Caddyfile
 ```
 
 See the [module README](https://github.com/unlabs-dev/caddy-uncloud#usage) for custom image builds, Compose deployment,
@@ -55,4 +64,10 @@ List issued certificates in cluster storage with [`uc caddy cert ls`](../../9-cl
 
 ```shell
 uc caddy cert ls
+
+ID             NAME                            ISSUER                    EXPIRES
+c111e23615f7   dns.uncloud.run                 Let's Encrypt             2026-12-31 (2 months)
+b5dcd2a03e1b   nginx.2t5ex2.uncld.dev          Let's Encrypt             2026-12-31 (2 months)
+70707fd2fea2   test-staging.2t5ex2.uncld.dev   Let's Encrypt (staging)   2026-12-31 (2 months)
+c5f9301e1c06   uncloud.run                     Let's Encrypt             2026-12-31 (2 months)
 ```

website/docs/3-concepts/6-services/1-internal-dns.md

@@ -3,6 +3,7 @@
 Services can be addressed on the internal WireGuard network by service name, service ID, or a machine-scoped service name:
 
 ## Service name
+
 ```
 $ nslookup nats.internal
 Server:         127.0.0.11
@@ -30,6 +31,7 @@ Address: 10.210.1.4
 ```
 
 ## Service ID
+
 ```
 $ nslookup 3ecb3a8bbec5fd3f46efb056a934714a.internal
 Server:         127.0.0.11
@@ -40,6 +42,7 @@ Address: 10.210.0.4
 ```
 
 ## Machine ID scoped service name
+
 ```
 $ nslookup 0903f0ee483aa97d559eeeaac5e22283.m.nats.internal
 Server:         127.0.0.11
@@ -64,7 +67,8 @@ Address: 10.210.1.4
 
 Additionally, the IP ordering preference can be specified with a `rr` (round-robin) or `nearest` subdomain prefix.
 
-### `rr` (round-robin) *current default*
+### `rr` (round-robin) _current default_
+
 Randomly shuffled order on each lookup.
 
 ```
@@ -96,9 +100,11 @@ Address: 10.210.0.3
 ```
 
 ## Nearest scope
+
 Returns machine-local instances first.
 
 `machine-a`:
+
 ```
 $ nslookup nearest.worker.internal
 Server:         127.0.0.11
@@ -115,6 +121,7 @@ Address: 10.210.1.4
 ```
 
 `machine-b`:
+
 ```
 $ nslookup nearest.worker.internal
 Server:         127.0.0.11
@@ -131,3 +138,28 @@ Address: 10.210.0.4
 ```
 
 The prefixes can be used with service ID and machine-scoped service names, as well (e.g. `nearest.3ecb3a8bbec5fd3f46efb056a934714a.internal` or `rr.0903f0ee483aa97d559eeeaac5e22283.m.worker.internal`).
+
+## Machine name
+
+Machines can be addressed too on the internal WireGuard network, either by name of by ID, these names are
+available from the `m.internal` zone.
+
+```
+$ nslookup machine-1.m.internal
+Server:         127.0.0.11
+Address:        127.0.0.11#53
+
+Name:   machine-1.m.internal
+Address: 10.210.0.1
+```
+
+## Machine ID
+
+```
+$ nslookup c337f00600de51ef4375c9a9a267dba5.m.internal
+Server:         127.0.0.11
+Address:        127.0.0.11#53
+
+Name:   c337f00600de51ef4375c9a9a267dba5.m.internal
+Address: 10.210.0.1
+```

website/docs/3-concepts/8-secrets.md

@@ -12,8 +12,8 @@ values at deploy time from your password manager, a cloud secrets store, or loca
 Two things go into your Compose file:
 
 1. Define a secret in the top-level `secrets` section. This tells Uncloud how to get the secret value.
-2. Reference that secret value in a service's `environment` using the `secret://<name>` format. This tells Uncloud to
-   resolve the secret and set the environment variable to its value at deploy time.
+2. Reference that secret value in a service's `environment` or `x-pre_deploy.environment` using the `secret://<name>`
+   format. This tells Uncloud to resolve the secret and set the environment variable to its value at deploy time.
 
 ```yaml title="compose.yaml"
 services:

website/docs/8-compose-file-reference/2-extensions.md

@@ -105,13 +105,13 @@ services:
 
 ### Attributes
 
-| Attribute     | Type                    | Default         | Description                                                                                                                                                        |
-|---------------|-------------------------|-----------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `command`     | string / list           | (required)      | The command to run in the hook container (same format as the service's [`command`](https://github.com/compose-spec/compose-spec/blob/main/05-services.md#command)) |
-| `environment` | map / list of KEY=VALUE | -               | Additional env vars that override or extend the service's [`environment`](https://github.com/compose-spec/compose-spec/blob/main/05-services.md#environment)       |
-| `privileged`  | bool                    | service's value | Override the service's [`privileged`](https://github.com/compose-spec/compose-spec/blob/main/05-services.md#privileged) mode                                       |
-| `timeout`     | duration                | `5m`            | Max time to wait for the command to finish before killing it (e.g., `1m30s`, `30m`, `1h`)                                                                          |
-| `user`        | string                  | service's value | Override the service's [`user`](https://github.com/compose-spec/compose-spec/blob/main/05-services.md#user) to run as (`user`, `UID`, `user:group`, or `UID:GID`)  |
+| Attribute     | Type                    | Default         | Description                                                                                                                                                                                                                                     |
+|---------------|-------------------------|-----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `command`     | string / list           | (required)      | The command to run in the hook container (same format as the service's [`command`](https://github.com/compose-spec/compose-spec/blob/main/05-services.md#command))                                                                              |
+| `environment` | map / list of KEY=VALUE | -               | Additional env vars that override or extend the service's [`environment`](https://github.com/compose-spec/compose-spec/blob/main/05-services.md#environment). Supports `secret://<name>` references. See [Secrets](../3-concepts/8-secrets.md). |
+| `privileged`  | bool                    | service's value | Override the service's [`privileged`](https://github.com/compose-spec/compose-spec/blob/main/05-services.md#privileged) mode                                                                                                                    |
+| `timeout`     | duration                | `5m`            | Max time to wait for the command to finish before killing it (e.g., `1m30s`, `30m`, `1h`)                                                                                                                                                       |
+| `user`        | string                  | service's value | Override the service's [`user`](https://github.com/compose-spec/compose-spec/blob/main/05-services.md#user) to run as (`user`, `UID`, `user:group`, or `UID:GID`)                                                                               |
 
 The hook container also gets `UNCLOUD_HOOK_PRE_DEPLOY=true` environment variable set automatically.
 

website/docs/9-cli-reference/uc_caddy_config.md

@@ -15,7 +15,6 @@ uc caddy config [flags]
 ```
   -h, --help             help for config
   -m, --machine string   Name or ID of the machine to get the configuration from. (default is connected machine)
-      --no-color         Disable syntax highlighting for the output.
 ```
 
 ## Options inherited from parent commands

website/docs/9-cli-reference/uc_caddy_logs.md

@@ -21,13 +21,16 @@ uc caddy logs [flags]
   -m, --machine strings   Filter logs by machine name or ID. Can be specified multiple times or as a comma-separated list.
       --since string      Show logs generated on or after the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.
                           Examples:
-                            --since 2m30s                      Relative duration (2 minutes 30 seconds ago)
-                            --since 1h                         Relative duration (1 hour ago)
-                            --since 2025-11-24                 RFC 3339 date only (midnight using local timezone)
-                            --since 2024-05-14T22:50:00        RFC 3339 date/time using local timezone
+                            --since 1h45m                      Relative duration (1 hour 45 minutes ago)
+                                                               Supported units: d (day = 24h), h (hour), m (minute),
+                                                                                s (second), ms (millisecond),
+                                                                                us/µs (microsecond), ns (nanosecond)
+                            --since 2025-11-24                 RFC 3339 date only (midnight using client local timezone)
+                            --since 2024-05-14T22:50:00        RFC 3339 date/time using client local timezone
                             --since 2024-01-31T10:30:00Z       RFC 3339 date/time in UTC
                             --since 1763953966                 Unix timestamp (seconds since January 1, 1970)
-  -n, --tail string       Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs. (default "100")
+  -n, --tail string       Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs.
+                          Defaults to 100, or 'all' when --since is set.
       --until string      Show logs generated before the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.
                           See --since for examples.
       --utc               Print timestamps in UTC instead of local timezone.

website/docs/9-cli-reference/uc_logs.md

@@ -56,13 +56,16 @@ uc logs [SERVICE[/CONTAINER]...] [flags]
   -m, --machine strings   Filter logs by machine name or ID. Can be specified multiple times or as a comma-separated list.
       --since string      Show logs generated on or after the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.
                           Examples:
-                            --since 2m30s                      Relative duration (2 minutes 30 seconds ago)
-                            --since 1h                         Relative duration (1 hour ago)
-                            --since 2025-11-24                 RFC 3339 date only (midnight using local timezone)
-                            --since 2024-05-14T22:50:00        RFC 3339 date/time using local timezone
+                            --since 1h45m                      Relative duration (1 hour 45 minutes ago)
+                                                               Supported units: d (day = 24h), h (hour), m (minute),
+                                                                                s (second), ms (millisecond),
+                                                                                us/µs (microsecond), ns (nanosecond)
+                            --since 2025-11-24                 RFC 3339 date only (midnight using client local timezone)
+                            --since 2024-05-14T22:50:00        RFC 3339 date/time using client local timezone
                             --since 2024-01-31T10:30:00Z       RFC 3339 date/time in UTC
                             --since 1763953966                 Unix timestamp (seconds since January 1, 1970)
-  -n, --tail string       Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs. (default "100")
+  -n, --tail string       Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs.
+                          Defaults to 100, or 'all' when --since is set.
       --until string      Show logs generated before the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.
                           See --since for examples.
       --utc               Print timestamps in UTC instead of local timezone.

website/docs/9-cli-reference/uc_machine_logs.md

@@ -52,13 +52,16 @@ uc machine logs [SERVICE...] [flags]
   -m, --machine strings   Filter logs by machine name or ID. Can be specified multiple times or as a comma-separated list.
       --since string      Show logs generated on or after the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.
                           Examples:
-                            --since 2m30s                      Relative duration (2 minutes 30 seconds ago)
-                            --since 1h                         Relative duration (1 hour ago)
-                            --since 2025-11-24                 RFC 3339 date only (midnight using local timezone)
-                            --since 2024-05-14T22:50:00        RFC 3339 date/time using local timezone
+                            --since 1h45m                      Relative duration (1 hour 45 minutes ago)
+                                                               Supported units: d (day = 24h), h (hour), m (minute),
+                                                                                s (second), ms (millisecond),
+                                                                                us/µs (microsecond), ns (nanosecond)
+                            --since 2025-11-24                 RFC 3339 date only (midnight using client local timezone)
+                            --since 2024-05-14T22:50:00        RFC 3339 date/time using client local timezone
                             --since 2024-01-31T10:30:00Z       RFC 3339 date/time in UTC
                             --since 1763953966                 Unix timestamp (seconds since January 1, 1970)
-  -n, --tail string       Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs. (default "100")
+  -n, --tail string       Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs.
+                          Defaults to 100, or 'all' when --since is set.
       --until string      Show logs generated before the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.
                           See --since for examples.
       --utc               Print timestamps in UTC instead of local timezone.

website/docs/9-cli-reference/uc_service_logs.md

@@ -56,13 +56,16 @@ uc service logs [SERVICE[/CONTAINER]...] [flags]
   -m, --machine strings   Filter logs by machine name or ID. Can be specified multiple times or as a comma-separated list.
       --since string      Show logs generated on or after the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.
                           Examples:
-                            --since 2m30s                      Relative duration (2 minutes 30 seconds ago)
-                            --since 1h                         Relative duration (1 hour ago)
-                            --since 2025-11-24                 RFC 3339 date only (midnight using local timezone)
-                            --since 2024-05-14T22:50:00        RFC 3339 date/time using local timezone
+                            --since 1h45m                      Relative duration (1 hour 45 minutes ago)
+                                                               Supported units: d (day = 24h), h (hour), m (minute),
+                                                                                s (second), ms (millisecond),
+                                                                                us/µs (microsecond), ns (nanosecond)
+                            --since 2025-11-24                 RFC 3339 date only (midnight using client local timezone)
+                            --since 2024-05-14T22:50:00        RFC 3339 date/time using client local timezone
                             --since 2024-01-31T10:30:00Z       RFC 3339 date/time in UTC
                             --since 1763953966                 Unix timestamp (seconds since January 1, 1970)
-  -n, --tail string       Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs. (default "100")
+  -n, --tail string       Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs.
+                          Defaults to 100, or 'all' when --since is set.
       --until string      Show logs generated before the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.
                           See --since for examples.
       --utc               Print timestamps in UTC instead of local timezone.