feat: service logs command to stream service logs (#196)

* chore: fix landing logo shadow

* feat: implement service logs command with colored output and strict ordering

* revert Makefile

* fix after rebase

* move logs command under services with root shortcut

* simplify server options for streaming logs and CLI flags

* refactor ContainerLogs grpc server

* update --tail flag

* minor docker.proto

* refactor client ServiceLogs and ContainerLogs

* move ProxyMachinesContext from api to client pkg

* minor refactor proto Stream

* add api/logs

* implement LogMerger

* fix LogMerger to correctly use semaphore

* increate inflish entries to 100 per stream

* send heartbeats

* refactor ContainerLogs to synchronise Send of entries and heartbeats to the steram

* minor logmerege

* remove ContainerName form ServiceLogEntryMetadata

* refactor ContainerLogs into docker.Service

* detect stalled container logs streams

* update logmerger tests

* fix comment in test

* refactor LogMerger with options

* uc logs: format one or multiple services

* make LogMerger emit heartbeats, emit entries <= watermark, rewrite tests

* update uc logs with new LogMerger

* go mod tidy

* fix after merge

---------

Co-authored-by: Evgenii Orlov <evgenii.orlov@semrush.com>

cmd/uncloud/caddy/config.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/alecthomas/chroma/v2/quick"
 	"github.com/psviderski/uncloud/internal/cli"
-	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/spf13/cobra"
 )
 
@@ -46,7 +45,7 @@ func runConfig(ctx context.Context, uncli *cli.CLI, opts configOptions) error {
 
 	if opts.machine != "" {
 		// If a specific machine is requested, use it to get the Caddy configuration.
-		ctx, _, err = api.ProxyMachinesContext(ctx, clusterClient, []string{opts.machine})
+		ctx, _, err = clusterClient.ProxyMachinesContext(ctx, []string{opts.machine})
 		if err != nil {
 			return err
 		}

cmd/uncloud/machine/rm.go

@@ -55,7 +55,7 @@ func remove(ctx context.Context, uncli *cli.CLI, nameOrID string, opts removeOpt
 	defer client.Close()
 
 	// Verify the machine exists and list all service containers on it including stopped ones.
-	mctx, machines, err := api.ProxyMachinesContext(ctx, client, []string{nameOrID})
+	mctx, machines, err := client.ProxyMachinesContext(ctx, []string{nameOrID})
 	if err != nil {
 		return err
 	}

cmd/uncloud/machine/root.go

@@ -8,7 +8,7 @@ func NewRootCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "machine",
 		Aliases: []string{"m"},
-		Short:   "Manage machines in an Uncloud cluster.",
+		Short:   "Manage machines in the cluster.",
 	}
 	cmd.AddCommand(
 		NewAddCommand(),

cmd/uncloud/main.go

@@ -108,9 +108,9 @@ func main() {
 	})
 
 	cmd.AddCommand(
+		NewBuildCommand(),
 		NewDeployCommand(),
 		NewDocsCommand(),
-		NewBuildCommand(),
 		NewImagesCommand(),
 		NewPsCommand(),
 		caddy.NewRootCommand(),
@@ -122,6 +122,7 @@ func main() {
 		service.NewExecCommand(),
 		service.NewInspectCommand(),
 		service.NewListCommand(),
+		service.NewLogsCommand(),
 		service.NewRmCommand(),
 		service.NewRunCommand(),
 		service.NewScaleCommand(),

cmd/uncloud/ps.go

@@ -12,7 +12,6 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/psviderski/uncloud/internal/cli"
-	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client"
 )
 
@@ -46,12 +45,14 @@ This command provides a comprehensive overview of all running containers that ar
 making it easy to see the distribution and status of containers across the cluster.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if opts.sortBy != sortByService && opts.sortBy != sortByMachine && opts.sortBy != sortByHealth {
-				return fmt.Errorf("invalid value for --sort: %q, must be one of '%s', '%s' or '%s'", opts.sortBy, sortByService, sortByMachine, sortByHealth)
+				return fmt.Errorf("invalid value for --sort: %q, must be one of '%s', '%s' or '%s'", opts.sortBy,
+					sortByService, sortByMachine, sortByHealth)
 			}
 			return runPs(cmd, opts)
 		},
 	}
-	cmd.Flags().StringVarP(&opts.sortBy, "sort", "s", sortByService, "Sort containers by 'service', 'machine' or 'health'")
+	cmd.Flags().StringVarP(&opts.sortBy, "sort", "s", sortByService,
+		"Sort containers by 'service', 'machine' or 'health'")
 	return cmd
 }
 
@@ -176,7 +177,7 @@ func printContainers(containers []containerInfo) error {
 }
 
 func collectContainers(ctx context.Context, cli *client.Client) ([]containerInfo, error) {
-	listCtx, machines, err := api.ProxyMachinesContext(ctx, cli, nil)
+	listCtx, machines, err := cli.ProxyMachinesContext(ctx, nil)
 	if err != nil {
 		return nil, fmt.Errorf("proxy machines context: %w", err)
 	}
@@ -206,7 +207,8 @@ func collectContainers(ctx context.Context, cli *client.Client) ([]containerInfo
 			return nil, fmt.Errorf("something went wrong with gRPC proxy: metadata is missing for a machine response")
 		}
 		if msc.Metadata != nil && msc.Metadata.Error != "" {
-			client.PrintWarning(fmt.Sprintf("failed to list containers on machine %s: %s", machineName, msc.Metadata.Error))
+			client.PrintWarning(fmt.Sprintf("failed to list containers on machine %s: %s", machineName,
+				msc.Metadata.Error))
 			continue
 		}
 

cmd/uncloud/service/logs.go

@@ -0,0 +1,279 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/charmbracelet/lipgloss"
+	mapset "github.com/deckarep/golang-set/v2"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/psviderski/uncloud/internal/cli"
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/psviderski/uncloud/pkg/client"
+	"github.com/spf13/cobra"
+)
+
+type logsOptions struct {
+	follow bool
+	tail   string
+	since  string
+	until  string
+}
+
+func NewLogsCommand() *cobra.Command {
+	var options logsOptions
+
+	cmd := &cobra.Command{
+		Use:     "logs SERVICE [SERVICE...]",
+		Aliases: []string{"log"},
+		Short:   "View service logs.",
+		Long:    "View logs from all replicas of the specified service(s) across all machines in the cluster.",
+		Args:    cobra.MinimumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			uncli := cmd.Context().Value("cli").(*cli.CLI)
+			return streamLogs(cmd.Context(), uncli, args, options)
+		},
+	}
+
+	cmd.Flags().BoolVarP(&options.follow, "follow", "f", false,
+		"Continually stream new logs.")
+	cmd.Flags().StringVarP(&options.tail, "tail", "n", "100",
+		"Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs.")
+	cmd.Flags().StringVar(&options.since, "since", "",
+		"Show logs generated on or after the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.\n"+
+			"Examples:\n"+
+			"  --since 2m30s                      Relative duration (2 minutes 30 seconds ago)\n"+
+			"  --since 1h                         Relative duration (1 hour ago)\n"+
+			"  --since 2025-11-24                 RFC 3339 date only (midnight using local timezone)\n"+
+			"  --since 2024-05-14T22:50:00        RFC 3339 date/time using local timezone\n"+
+			"  --since 2024-01-31T10:30:00Z       RFC 3339 date/time in UTC\n"+
+			"  --since 1763953966                 Unix timestamp (seconds since January 1, 1970)")
+	cmd.Flags().StringVar(&options.until, "until", "",
+		"Show logs generated before the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.\n"+
+			"See --since for examples.")
+
+	return cmd
+}
+
+func streamLogs(ctx context.Context, uncli *cli.CLI, serviceNames []string, opts logsOptions) error {
+	// Parse tail option.
+	tail := -1
+	if opts.tail != "all" {
+		tailInt, err := strconv.Atoi(opts.tail)
+		if err != nil {
+			return fmt.Errorf("invalid --tail value '%s': %w", opts.tail, err)
+		}
+		tail = tailInt
+	}
+
+	c, err := uncli.ConnectCluster(ctx)
+	if err != nil {
+		return fmt.Errorf("connect to cluster: %w", err)
+	}
+	defer c.Close()
+
+	logsOpts := api.ServiceLogsOptions{
+		Follow: opts.follow,
+		Tail:   tail,
+		Since:  opts.since,
+		Until:  opts.until,
+	}
+
+	// Collect log streams from all services.
+	machineIDsSet := mapset.NewSet[string]()
+	svcStreams := make([]<-chan api.ServiceLogEntry, 0, len(serviceNames))
+	for _, serviceName := range serviceNames {
+		// TODO: set Heartbeats in the opts.
+		svc, ch, err := c.ServiceLogs(ctx, serviceName, logsOpts)
+		if err != nil {
+			return fmt.Errorf("stream logs for service '%s': %w", serviceName, err)
+		}
+		svcStreams = append(svcStreams, ch)
+
+		machineIDs := svc.MachineIDs()
+		machineIDsSet.Append(machineIDs...)
+	}
+
+	var stream <-chan api.ServiceLogEntry
+	if len(serviceNames) == 1 {
+		stream = svcStreams[0]
+	} else {
+		// Merge all service streams into a single sorted stream without stall detection as its handled per-service.
+		merger := client.NewLogMerger(svcStreams, client.LogMergerOptions{})
+		stream = merger.Stream()
+	}
+
+	// Fetch machine names for all machines (machineIDsSet) service containers are running on.
+	machines, err := c.ListMachines(ctx, &api.MachineFilter{NamesOrIDs: machineIDsSet.ToSlice()})
+	if err != nil {
+		return fmt.Errorf("list machines: %w", err)
+	}
+	machineNames := make([]string, 0, len(machines))
+	for _, m := range machines {
+		machineNames = append(machineNames, m.Machine.Name)
+	}
+
+	formatter := newLogFormatter(machineNames, serviceNames)
+
+	// Print merged logs.
+	for entry := range stream {
+		if entry.Err != nil {
+			formatter.printError(entry)
+			continue
+		}
+		formatter.printEntry(entry)
+	}
+
+	return nil
+}
+
+// Available colors for machine/service differentiation.
+var colorPalette = []lipgloss.Color{
+	lipgloss.Color("10"), // Bright green
+	lipgloss.Color("11"), // Bright yellow
+	lipgloss.Color("12"), // Bright blue
+	lipgloss.Color("13"), // Bright magenta
+	lipgloss.Color("14"), // Bright cyan
+	lipgloss.Color("2"),  // Green
+	lipgloss.Color("3"),  // Yellow
+	lipgloss.Color("4"),  // Blue
+	lipgloss.Color("5"),  // Magenta
+	lipgloss.Color("6"),  // Cyan
+}
+
+// logFormatter handles formatting and printing of log entries with dynamic column alignment.
+type logFormatter struct {
+	machineNames []string
+	serviceNames []string
+
+	maxMachineWidth int
+	maxServiceWidth int
+}
+
+func newLogFormatter(machineNames, serviceNames []string) *logFormatter {
+	slices.Sort(machineNames)
+	slices.Sort(serviceNames)
+
+	maxMachineWidth := 0
+	for _, name := range machineNames {
+		if len(name) > maxMachineWidth {
+			maxMachineWidth = len(name)
+		}
+	}
+
+	maxServiceWidth := 0
+	for _, name := range serviceNames {
+		if len(name) > maxServiceWidth {
+			maxServiceWidth = len(name)
+		}
+	}
+
+	return &logFormatter{
+		machineNames:    machineNames,
+		serviceNames:    serviceNames,
+		maxMachineWidth: maxMachineWidth,
+		maxServiceWidth: maxServiceWidth,
+	}
+}
+
+// formatTimestamp formats timestamp using local timezone.
+func (f *logFormatter) formatTimestamp(t time.Time) string {
+	dimStyle := lipgloss.NewStyle().Faint(true)
+	t = t.In(time.Local)
+	return dimStyle.Render(t.Format(time.StampMilli))
+}
+
+func (f *logFormatter) formatMachine(name string) string {
+	style := lipgloss.NewStyle().Bold(true).PaddingRight(f.maxMachineWidth - len(name))
+
+	if len(f.serviceNames) == 1 {
+		// Machine name is coloured for single-service logs.
+		i := slices.Index(f.machineNames, name)
+		if i == -1 {
+			f.machineNames = append(f.machineNames, name)
+			i = len(f.machineNames) - 1
+		}
+
+		style = style.Foreground(colorPalette[i%len(colorPalette)])
+	}
+
+	return style.Render(name)
+}
+
+func (f *logFormatter) formatServiceContainer(serviceName, containerID string) string {
+	styleService := lipgloss.NewStyle().Bold(true).PaddingRight(f.maxServiceWidth - len(serviceName))
+	styleContainer := lipgloss.NewStyle().Faint(true)
+
+	if len(f.serviceNames) > 1 {
+		// Service name is coloured for multi-service logs.
+		i := slices.Index(f.serviceNames, serviceName)
+		if i == -1 {
+			f.serviceNames = append(f.serviceNames, serviceName)
+			i = len(f.serviceNames) - 1
+		}
+
+		styleService = styleService.Foreground(colorPalette[i%len(colorPalette)])
+	}
+
+	return styleService.Render(serviceName) + styleContainer.Render("["+containerID[:5]+"]")
+}
+
+// printEntry prints a single log entry with proper formatting.
+func (f *logFormatter) printEntry(entry api.ServiceLogEntry) {
+	if entry.Stream != api.LogStreamStdout && entry.Stream != api.LogStreamStderr {
+		return
+	}
+
+	var output strings.Builder
+
+	// Timestamp
+	output.WriteString(f.formatTimestamp(entry.Timestamp))
+	output.WriteString(" ")
+
+	// Machine name
+	output.WriteString(f.formatMachine(entry.Metadata.MachineName))
+	output.WriteString(" ")
+
+	// Service[container_id]
+	output.WriteString(f.formatServiceContainer(entry.Metadata.ServiceName, entry.Metadata.ContainerID))
+	output.WriteString(" ")
+
+	// Message
+	output.Write(entry.Message)
+
+	// Print to appropriate stream.
+	if entry.Stream == api.LogStreamStderr {
+		fmt.Fprint(os.Stderr, output.String())
+	} else {
+		fmt.Print(output.String())
+	}
+}
+
+// printError prints an error entry (e.g., stalled stream warning).
+func (f *logFormatter) printError(entry api.ServiceLogEntry) {
+	if entry.Metadata.ContainerID != "" {
+		msg := fmt.Sprintf("WARNING: log stream from %s[%s] on machine '%s'",
+			entry.Metadata.ServiceName,
+			stringid.TruncateID(entry.Metadata.ContainerID),
+			entry.Metadata.MachineName)
+
+		if errors.Is(entry.Err, api.ErrLogStreamStalled) {
+			msg += " stopped responding"
+		} else {
+			msg += fmt.Sprintf(": %v", entry.Err)
+		}
+
+		style := lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("11")) // Bold bright yellow
+		fmt.Fprintln(os.Stderr, style.Render(msg))
+	} else {
+		msg := fmt.Sprintf("ERROR: %v", entry.Err)
+		style := lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("9")) // Bold bright red
+		fmt.Fprintln(os.Stderr, style.Render(msg))
+	}
+}

cmd/uncloud/service/root.go

@@ -8,15 +8,16 @@ func NewRootCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "service",
 		Aliases: []string{"svc"},
-		Short:   "Manage services in an Uncloud cluster.",
+		Short:   "Manage services in the cluster.",
 	}
 	cmd.AddCommand(
+		NewExecCommand(),
 		NewInspectCommand(),
 		NewListCommand(),
+		NewLogsCommand(),
 		NewRmCommand(),
 		NewRunCommand(),
 		NewScaleCommand(),
-		NewExecCommand(),
 	)
 	return cmd
 }

cmd/uncloud/volume/root.go

@@ -7,7 +7,7 @@ import (
 func NewRootCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "volume",
-		Short: "Manage volumes in an Uncloud cluster.",
+		Short: "Manage volumes in the cluster.",
 	}
 	cmd.AddCommand(
 		NewCreateCommand(),

go.mod

@@ -150,7 +150,7 @@ require (
 	github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
-	github.com/fatih/color v1.17.0 // indirect
+	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsevents v0.2.0 // indirect
 	github.com/fvbommel/sortorder v1.1.0 // indirect

go.sum

@@ -346,8 +346,8 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
-github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
-github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=

internal/machine/api/pb/docker.proto

@@ -5,6 +5,7 @@ package api;
 option go_package = "github.com/psviderski/uncloud/internal/machine/api/pb";
 
 import "google/protobuf/empty.proto";
+import "google/protobuf/timestamp.proto";
 import "internal/machine/api/pb/common.proto";
 
 service Docker {
@@ -15,6 +16,9 @@ service Docker {
   rpc ListContainers(ListContainersRequest) returns (ListContainersResponse);
   rpc RemoveContainer(RemoveContainerRequest) returns (google.protobuf.Empty);
 
+  rpc ExecContainer(stream ExecContainerRequest) returns (stream ExecContainerResponse);
+  rpc ContainerLogs(ContainerLogsRequest) returns (stream ContainerLogEntry);
+
   rpc PullImage(PullImageRequest) returns (stream JSONMessage);
   rpc InspectImage(InspectImageRequest) returns (InspectImageResponse);
   // InspectRemoteImage returns the image metadata for an image in a remote registry using the machine's
@@ -30,8 +34,6 @@ service Docker {
   rpc InspectServiceContainer(InspectContainerRequest) returns (ServiceContainer);
   rpc ListServiceContainers(ListServiceContainersRequest) returns (ListServiceContainersResponse);
   rpc RemoveServiceContainer(RemoveContainerRequest) returns (google.protobuf.Empty);
-
-  rpc ExecContainer(stream ExecContainerRequest) returns (stream ExecContainerResponse);
 }
 
 message CreateContainerRequest {
@@ -94,6 +96,64 @@ message RemoveContainerRequest {
   bytes options = 2;
 }
 
+message ExecContainerRequest {
+  oneof payload {
+    // Initial configuration for the exec session. Must be sent as the first message.
+    ExecConfig config = 1;
+    // Raw stdin data to be written to the exec process.
+    bytes stdin = 2;
+    // TTY resize event (only used when TTY is enabled).
+    ResizeEvent resize = 3;
+  }
+}
+
+message ExecConfig {
+  // Container ID to execute the command in.
+  string container_id = 1;
+  // JSON serialised ExecOptions
+  bytes options = 2;
+}
+
+message ResizeEvent {
+  uint32 height = 1;
+  uint32 width = 2;
+}
+
+message ExecContainerResponse {
+  oneof payload {
+    // Exec instance ID returned after creating the exec.
+    string exec_id = 1;
+    // Raw stdout data from the exec process.
+    bytes stdout = 2;
+    // Raw stderr data from the exec process (only when TTY is disabled).
+    bytes stderr = 3;
+    // Exit code of the exec process. Sent as the final message.
+    int32 exit_code = 4;
+  }
+}
+
+message ContainerLogsRequest {
+  string container_id = 1;
+  // Options for logs retrieval.
+  bool follow = 2;
+  int32 tail = 3; // -1 means all
+  string since = 4; // https://www.rfc-editor.org/rfc/rfc3339.html timestamp or Go duration string
+  string until = 5; // https://www.rfc-editor.org/rfc/rfc3339.html timestamp or Go duration string
+}
+
+message ContainerLogEntry {
+  enum StreamType {
+    UNKNOWN = 0;
+    STDOUT = 1;
+    STDERR = 2;
+    HEARTBEAT = 3;
+  }
+  StreamType stream = 1;
+  google.protobuf.Timestamp timestamp = 2;
+  // Log line content. Empty for heartbeat entries.
+  bytes message = 3;
+}
+
 message PullImageRequest {
   string image = 1;
   // JSON serialised image.PullOptions.
@@ -216,39 +276,3 @@ message MachineServiceContainers {
   Metadata metadata = 1;
   repeated ServiceContainer containers = 2;
 }
-
-message ExecContainerRequest {
-  oneof payload {
-    // Initial configuration for the exec session. Must be sent as the first message.
-    ExecConfig config = 1;
-    // Raw stdin data to be written to the exec process.
-    bytes stdin = 2;
-    // TTY resize event (only used when TTY is enabled).
-    ResizeEvent resize = 3;
-  }
-}
-
-message ExecConfig {
-  // Container ID to execute the command in.
-  string container_id = 1;
-  // JSON serialised ExecOptions
-  bytes options = 2;
-}
-
-message ResizeEvent {
-  uint32 height = 1;
-  uint32 width = 2;
-}
-
-message ExecContainerResponse {
-  oneof payload {
-    // Exec instance ID returned after creating the exec.
-    string exec_id = 1;
-    // Raw stdout data from the exec process.
-    bytes stdout = 2;
-    // Raw stderr data from the exec process (only when TTY is disabled).
-    bytes stderr = 3;
-    // Exit code of the exec process. Sent as the final message.
-    int32 exit_code = 4;
-  }
-}

internal/machine/api/pb/docker_grpc.pb.go

@@ -26,6 +26,8 @@ const (
 	Docker_StopContainer_FullMethodName           = "/api.Docker/StopContainer"
 	Docker_ListContainers_FullMethodName          = "/api.Docker/ListContainers"
 	Docker_RemoveContainer_FullMethodName         = "/api.Docker/RemoveContainer"
+	Docker_ExecContainer_FullMethodName           = "/api.Docker/ExecContainer"
+	Docker_ContainerLogs_FullMethodName           = "/api.Docker/ContainerLogs"
 	Docker_PullImage_FullMethodName               = "/api.Docker/PullImage"
 	Docker_InspectImage_FullMethodName            = "/api.Docker/InspectImage"
 	Docker_InspectRemoteImage_FullMethodName      = "/api.Docker/InspectRemoteImage"
@@ -37,7 +39,6 @@ const (
 	Docker_InspectServiceContainer_FullMethodName = "/api.Docker/InspectServiceContainer"
 	Docker_ListServiceContainers_FullMethodName   = "/api.Docker/ListServiceContainers"
 	Docker_RemoveServiceContainer_FullMethodName  = "/api.Docker/RemoveServiceContainer"
-	Docker_ExecContainer_FullMethodName           = "/api.Docker/ExecContainer"
 )
 
 // DockerClient is the client API for Docker service.
@@ -50,6 +51,8 @@ type DockerClient interface {
 	StopContainer(ctx context.Context, in *StopContainerRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
 	ListContainers(ctx context.Context, in *ListContainersRequest, opts ...grpc.CallOption) (*ListContainersResponse, error)
 	RemoveContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
+	ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error)
+	ContainerLogs(ctx context.Context, in *ContainerLogsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ContainerLogEntry], error)
 	PullImage(ctx context.Context, in *PullImageRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[JSONMessage], error)
 	InspectImage(ctx context.Context, in *InspectImageRequest, opts ...grpc.CallOption) (*InspectImageResponse, error)
 	// InspectRemoteImage returns the image metadata for an image in a remote registry using the machine's
@@ -63,7 +66,6 @@ type DockerClient interface {
 	InspectServiceContainer(ctx context.Context, in *InspectContainerRequest, opts ...grpc.CallOption) (*ServiceContainer, error)
 	ListServiceContainers(ctx context.Context, in *ListServiceContainersRequest, opts ...grpc.CallOption) (*ListServiceContainersResponse, error)
 	RemoveServiceContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
-	ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error)
 }
 
 type dockerClient struct {
@@ -134,9 +136,41 @@ func (c *dockerClient) RemoveContainer(ctx context.Context, in *RemoveContainerR
 	return out, nil
 }
 
+func (c *dockerClient) ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[0], Docker_ExecContainer_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[ExecContainerRequest, ExecContainerResponse]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ExecContainerClient = grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse]
+
+func (c *dockerClient) ContainerLogs(ctx context.Context, in *ContainerLogsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ContainerLogEntry], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[1], Docker_ContainerLogs_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[ContainerLogsRequest, ContainerLogEntry]{ClientStream: stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ContainerLogsClient = grpc.ServerStreamingClient[ContainerLogEntry]
+
 func (c *dockerClient) PullImage(ctx context.Context, in *PullImageRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[JSONMessage], error) {
 	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
-	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[0], Docker_PullImage_FullMethodName, cOpts...)
+	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[2], Docker_PullImage_FullMethodName, cOpts...)
 	if err != nil {
 		return nil, err
 	}
@@ -253,19 +287,6 @@ func (c *dockerClient) RemoveServiceContainer(ctx context.Context, in *RemoveCon
 	return out, nil
 }
 
-func (c *dockerClient) ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error) {
-	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
-	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[1], Docker_ExecContainer_FullMethodName, cOpts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &grpc.GenericClientStream[ExecContainerRequest, ExecContainerResponse]{ClientStream: stream}
-	return x, nil
-}
-
-// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
-type Docker_ExecContainerClient = grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse]
-
 // DockerServer is the server API for Docker service.
 // All implementations must embed UnimplementedDockerServer
 // for forward compatibility.
@@ -276,6 +297,8 @@ type DockerServer interface {
 	StopContainer(context.Context, *StopContainerRequest) (*emptypb.Empty, error)
 	ListContainers(context.Context, *ListContainersRequest) (*ListContainersResponse, error)
 	RemoveContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error)
+	ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error
+	ContainerLogs(*ContainerLogsRequest, grpc.ServerStreamingServer[ContainerLogEntry]) error
 	PullImage(*PullImageRequest, grpc.ServerStreamingServer[JSONMessage]) error
 	InspectImage(context.Context, *InspectImageRequest) (*InspectImageResponse, error)
 	// InspectRemoteImage returns the image metadata for an image in a remote registry using the machine's
@@ -289,7 +312,6 @@ type DockerServer interface {
 	InspectServiceContainer(context.Context, *InspectContainerRequest) (*ServiceContainer, error)
 	ListServiceContainers(context.Context, *ListServiceContainersRequest) (*ListServiceContainersResponse, error)
 	RemoveServiceContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error)
-	ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error
 	mustEmbedUnimplementedDockerServer()
 }
 
@@ -318,6 +340,12 @@ func (UnimplementedDockerServer) ListContainers(context.Context, *ListContainers
 func (UnimplementedDockerServer) RemoveContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method RemoveContainer not implemented")
 }
+func (UnimplementedDockerServer) ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method ExecContainer not implemented")
+}
+func (UnimplementedDockerServer) ContainerLogs(*ContainerLogsRequest, grpc.ServerStreamingServer[ContainerLogEntry]) error {
+	return status.Errorf(codes.Unimplemented, "method ContainerLogs not implemented")
+}
 func (UnimplementedDockerServer) PullImage(*PullImageRequest, grpc.ServerStreamingServer[JSONMessage]) error {
 	return status.Errorf(codes.Unimplemented, "method PullImage not implemented")
 }
@@ -351,9 +379,6 @@ func (UnimplementedDockerServer) ListServiceContainers(context.Context, *ListSer
 func (UnimplementedDockerServer) RemoveServiceContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method RemoveServiceContainer not implemented")
 }
-func (UnimplementedDockerServer) ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error {
-	return status.Errorf(codes.Unimplemented, "method ExecContainer not implemented")
-}
 func (UnimplementedDockerServer) mustEmbedUnimplementedDockerServer() {}
 func (UnimplementedDockerServer) testEmbeddedByValue()                {}
 
@@ -483,6 +508,24 @@ func _Docker_RemoveContainer_Handler(srv interface{}, ctx context.Context, dec f
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Docker_ExecContainer_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(DockerServer).ExecContainer(&grpc.GenericServerStream[ExecContainerRequest, ExecContainerResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ExecContainerServer = grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]
+
+func _Docker_ContainerLogs_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(ContainerLogsRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(DockerServer).ContainerLogs(m, &grpc.GenericServerStream[ContainerLogsRequest, ContainerLogEntry]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ContainerLogsServer = grpc.ServerStreamingServer[ContainerLogEntry]
+
 func _Docker_PullImage_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(PullImageRequest)
 	if err := stream.RecvMsg(m); err != nil {
@@ -674,13 +717,6 @@ func _Docker_RemoveServiceContainer_Handler(srv interface{}, ctx context.Context
 	return interceptor(ctx, in, info, handler)
 }
 
-func _Docker_ExecContainer_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(DockerServer).ExecContainer(&grpc.GenericServerStream[ExecContainerRequest, ExecContainerResponse]{ServerStream: stream})
-}
-
-// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
-type Docker_ExecContainerServer = grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]
-
 // Docker_ServiceDesc is the grpc.ServiceDesc for Docker service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -754,17 +790,22 @@ var Docker_ServiceDesc = grpc.ServiceDesc{
 		},
 	},
 	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "PullImage",
-			Handler:       _Docker_PullImage_Handler,
-			ServerStreams: true,
-		},
 		{
 			StreamName:    "ExecContainer",
 			Handler:       _Docker_ExecContainer_Handler,
 			ServerStreams: true,
 			ClientStreams: true,
 		},
+		{
+			StreamName:    "ContainerLogs",
+			Handler:       _Docker_ContainerLogs_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "PullImage",
+			Handler:       _Docker_PullImage_Handler,
+			ServerStreams: true,
+		},
 	},
 	Metadata: "internal/machine/api/pb/docker.proto",
 }

internal/machine/docker/server.go

@@ -47,6 +47,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/emptypb"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 var fullDockerIDRegex = regexp.MustCompile(`^[a-f0-9]{64}$`)
@@ -1015,6 +1016,97 @@ func (s *Server) RemoveServiceContainer(ctx context.Context, req *pb.RemoveConta
 	return resp, nil
 }
 
+// logsHeartbeatInterval is the interval at which heartbeat entries are sent when there are no logs to stream.
+const logsHeartbeatInterval = 200 * time.Millisecond
+
+// ContainerLogs streams logs from a container.
+func (s *Server) ContainerLogs(
+	req *pb.ContainerLogsRequest, stream grpc.ServerStreamingServer[pb.ContainerLogEntry],
+) error {
+	// Stream context is cancelled when the client has disconnected or the stream has ended.
+	ctx := stream.Context()
+
+	opts := ContainerLogsOptions{
+		ContainerID: req.ContainerId,
+		Follow:      req.Follow,
+		Tail:        int(req.Tail),
+		Since:       req.Since,
+		Until:       req.Until,
+	}
+
+	logsCh, err := s.service.ContainerLogs(ctx, opts)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return status.Error(codes.NotFound, err.Error())
+		}
+		return status.Errorf(codes.Internal, "get container logs: %v", err)
+	}
+
+	log := slog.With("container_id", req.ContainerId, "stream_id", fmt.Sprintf("%p", stream)[2:])
+	log.Debug("Starting container logs streaming.",
+		"follow", req.Follow, "tail", req.Tail, "since", req.Since, "until", req.Until)
+
+	// Heartbeats are needed only when following logs to let the client know when there are no new log entries
+	// to allow it to advance the watermark of last received log timestamp.
+	var heartbeatCh <-chan time.Time
+	if req.Follow {
+		heartbeatTicker := time.NewTicker(logsHeartbeatInterval)
+		defer heartbeatTicker.Stop()
+		heartbeatCh = heartbeatTicker.C
+	}
+
+	started := time.Now()
+	lastSent := time.Time{}
+
+	for {
+		select {
+		case entry, ok := <-logsCh:
+			if !ok {
+				// Channel closed, no more log entries.
+				return nil
+			}
+
+			if entry.Err != nil {
+				return status.Error(codes.Internal, entry.Err.Error())
+			}
+
+			pbEntry := &pb.ContainerLogEntry{
+				Stream:    api.LogStreamTypeToProto(entry.Stream),
+				Timestamp: timestamppb.New(entry.Timestamp),
+				Message:   entry.Message,
+			}
+			if err = stream.Send(pbEntry); err != nil {
+				return status.Errorf(codes.Internal, "send log entry: %v", err)
+			}
+			lastSent = entry.Timestamp
+
+		case now := <-heartbeatCh:
+			// Only send heartbeat if no log entries have been sent since the last heartbeat interval or
+			// if no log entries have been sent at all for at least a heartbeat interval since starting.
+			if now.Sub(lastSent) < logsHeartbeatInterval ||
+				(lastSent.IsZero() && now.Sub(started) < logsHeartbeatInterval) {
+				continue
+			}
+
+			// Use the timestamp one heartbeat in the past to be conservative. This reduces the chance of sending
+			// a timestamp that is greater than a log entry currently being parsed but not yet sent, which would
+			// cause the client to incorrectly believe it has received all logs up to that point.
+			heartbeat := &pb.ContainerLogEntry{
+				Stream:    pb.ContainerLogEntry_HEARTBEAT,
+				Timestamp: timestamppb.New(now.Add(-logsHeartbeatInterval)),
+			}
+			if err = stream.Send(heartbeat); err != nil {
+				return status.Errorf(codes.Internal, "send log stream heartbeat: %v", err)
+			}
+			lastSent = heartbeat.Timestamp.AsTime()
+			log.Debug("Sent log stream heartbeat.", "timestamp", lastSent)
+
+		case <-ctx.Done():
+			return status.Error(codes.Canceled, ctx.Err().Error())
+		}
+	}
+}
+
 // receiveExecConfig receives and validates the initial exec configuration from the stream.
 func (s *Server) receiveExecConfig(stream pb.Docker_ExecContainerServer) (*pb.ExecConfig, api.ExecOptions, error) {
 	req, err := stream.Recv()

internal/machine/docker/service.go

@@ -1,18 +1,22 @@
 package docker
 
 import (
+	"bytes"
 	"context"
 	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"log/slog"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/jmoiron/sqlx"
 	"github.com/psviderski/uncloud/pkg/api"
 	"google.golang.org/grpc/codes"
@@ -150,3 +154,104 @@ func (s *Service) ListImages(ctx context.Context, opts image.ListOptions) (Image
 
 	return imagesResp, nil
 }
+
+// ContainerLogsOptions specifies parameters for ContainerLogs.
+type ContainerLogsOptions struct {
+	ContainerID string
+	Follow      bool
+	Tail        int
+	Since       string
+	Until       string
+}
+
+// ContainerLogs streams logs from a container and returns demultiplexed entries via a channel.
+// The channel is closed when streaming completes or context is cancelled.
+func (s *Service) ContainerLogs(ctx context.Context, opts ContainerLogsOptions) (<-chan api.ContainerLogEntry, error) {
+	dockerOpts := container.LogsOptions{
+		ShowStdout: true,
+		ShowStderr: true,
+		Follow:     opts.Follow,
+		Tail:       strconv.FormatInt(int64(opts.Tail), 10),
+		Since:      opts.Since,
+		Until:      opts.Until,
+		Timestamps: true,
+	}
+
+	reader, err := s.Client.ContainerLogs(ctx, opts.ContainerID, dockerOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	outCh := make(chan api.ContainerLogEntry)
+	stdoutWriter := &logsChannelWriter{ctx: ctx, ch: outCh, isStderr: false}
+	stderrWriter := &logsChannelWriter{ctx: ctx, ch: outCh, isStderr: true}
+
+	// Wrap the context in a cancellable one to unblock the second goroutine below when StdCopy completes.
+	ctx, cancel := context.WithCancel(ctx)
+
+	// Run StdCopy in a goroutine to be able to handle context cancellation.
+	go func() {
+		defer close(outCh)
+		defer cancel()
+
+		// StdCopy is blocking and will return when the reader is closed in another goroutine below or on error.
+		if _, err := stdcopy.StdCopy(stdoutWriter, stderrWriter, reader); err != nil {
+			// Send error as the last entry.
+			select {
+			case outCh <- api.ContainerLogEntry{Err: fmt.Errorf("demultiplex container logs: %w", err)}:
+			case <-ctx.Done():
+			}
+		}
+	}()
+
+	// Close the reader when the context is done to cancel StdCopy if it's still running.
+	go func() {
+		<-ctx.Done()
+		reader.Close()
+	}()
+
+	return outCh, nil
+}
+
+// logsChannelWriter is a writer for stdcopy.StdCopy that sends demultiplexed container logs to a channel.
+type logsChannelWriter struct {
+	ctx      context.Context
+	ch       chan<- api.ContainerLogEntry
+	isStderr bool
+}
+
+func (w *logsChannelWriter) Write(data []byte) (n int, err error) {
+	// Parse timestamp and message from the demultiplexed Docker log payload if the data looks like it contains one.
+	// Format: 2025-01-01T00:00:00.000000000Z message
+	timestamp := time.Time{}
+	message := data
+	if len(data) > 30 && data[4] == '-' && data[7] == '-' && data[10] == 'T' {
+		timestampPart, messagePart, found := bytes.Cut(data, []byte(" "))
+		if found {
+			timestamp, err = time.Parse(time.RFC3339Nano, string(timestampPart))
+			if err != nil {
+				timestamp = time.Time{}
+			}
+			message = messagePart
+		}
+	}
+
+	entry := api.ContainerLogEntry{
+		Timestamp: timestamp,
+		// Clone is required because message is a slice into data, which stdcopy.StdCopy may reuse
+		// after Write returns but before the entry is consumed from the channel.
+		Message: bytes.Clone(message),
+	}
+	if w.isStderr {
+		entry.Stream = api.LogStreamStderr
+	} else {
+		entry.Stream = api.LogStreamStdout
+	}
+
+	select {
+	case w.ch <- entry:
+		return len(data), nil
+	case <-w.ctx.Done():
+		return 0, w.ctx.Err()
+	}
+}

pkg/api/client.go

@@ -2,13 +2,10 @@ package api
 
 import (
 	"context"
-	"fmt"
-	"strings"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/psviderski/uncloud/internal/machine/api/pb"
-	"google.golang.org/grpc/metadata"
 )
 
 type Client interface {
@@ -58,41 +55,3 @@ type VolumeClient interface {
 	ListVolumes(ctx context.Context, filter *VolumeFilter) ([]MachineVolume, error)
 	RemoveVolume(ctx context.Context, machineNameOrID, volumeName string, force bool) error
 }
-
-// ProxyMachinesContext returns a new context that proxies gRPC requests to the specified machines.
-// If namesOrIDs is nil, all machines are included.
-func ProxyMachinesContext(
-	ctx context.Context, cli MachineClient, namesOrIDs []string,
-) (context.Context, MachineMembersList, error) {
-	// TODO: move the machine IP resolution to the proxy router to allow setting machine names and IDs in the metadata.
-	machines, err := cli.ListMachines(ctx, nil)
-	if err != nil {
-		return nil, nil, fmt.Errorf("list machines: %w", err)
-	}
-
-	var proxiedMachines MachineMembersList
-	var notFound []string
-	for _, nameOrID := range namesOrIDs {
-		if m := machines.FindByNameOrID(nameOrID); m != nil {
-			proxiedMachines = append(proxiedMachines, m)
-		} else {
-			notFound = append(notFound, nameOrID)
-		}
-	}
-
-	if len(notFound) > 0 {
-		return nil, nil, fmt.Errorf("machines not found: %s", strings.Join(notFound, ", "))
-	}
-
-	if len(namesOrIDs) == 0 {
-		proxiedMachines = machines
-	}
-
-	md := metadata.New(nil)
-	for _, m := range proxiedMachines {
-		machineIP, _ := m.Machine.Network.ManagementIp.ToAddr()
-		md.Append("machines", machineIP.String())
-	}
-
-	return metadata.NewOutgoingContext(ctx, md), proxiedMachines, nil
-}

pkg/api/logs.go

@@ -0,0 +1,84 @@
+package api
+
+import (
+	"errors"
+	"time"
+
+	"github.com/psviderski/uncloud/internal/machine/api/pb"
+)
+
+const (
+	LogStreamUnknown LogStreamType = iota
+	LogStreamStdout
+	LogStreamStderr
+	// LogStreamHeartbeat represents a heartbeat log entry with a timestamp indicating that
+	// there are no older logs than this timestamp.
+	LogStreamHeartbeat
+)
+
+type LogStreamType int
+
+// LogStreamTypeFromProto converts a protobuf ContainerLogEntry.StreamType to the internal LogStreamType.
+func LogStreamTypeFromProto(s pb.ContainerLogEntry_StreamType) LogStreamType {
+	switch s {
+	case pb.ContainerLogEntry_STDOUT:
+		return LogStreamStdout
+	case pb.ContainerLogEntry_STDERR:
+		return LogStreamStderr
+	case pb.ContainerLogEntry_HEARTBEAT:
+		return LogStreamHeartbeat
+	default:
+		return LogStreamUnknown
+	}
+}
+
+// LogStreamTypeToProto converts LogStreamType to protobuf ContainerLogEntry.StreamType.
+func LogStreamTypeToProto(s LogStreamType) pb.ContainerLogEntry_StreamType {
+	switch s {
+	case LogStreamStdout:
+		return pb.ContainerLogEntry_STDOUT
+	case LogStreamStderr:
+		return pb.ContainerLogEntry_STDERR
+	case LogStreamHeartbeat:
+		return pb.ContainerLogEntry_HEARTBEAT
+	default:
+		return pb.ContainerLogEntry_UNKNOWN
+	}
+}
+
+// ServiceLogsOptions specifies parameters for ServiceLogs.
+type ServiceLogsOptions struct {
+	Follow bool
+	Tail   int
+	Since  string
+	Until  string
+}
+
+// ServiceLogEntry represents a single log entry from a service container.
+type ServiceLogEntry struct {
+	// Metadata may not be set if an error occurred (Err is not nil).
+	Metadata ServiceLogEntryMetadata
+	ContainerLogEntry
+}
+
+// ServiceLogEntryMetadata contains metadata about the source of a log entry.
+type ServiceLogEntryMetadata struct {
+	ServiceID   string
+	ServiceName string
+	ContainerID string
+	MachineID   string
+	MachineName string
+}
+
+// ContainerLogEntry represents a single log entry from a container.
+type ContainerLogEntry struct {
+	Stream    LogStreamType
+	Timestamp time.Time
+	Message   []byte
+	// Err indicates that an error occurred while streaming logs from a container.
+	// Other fields are not set if Err is not nil.
+	Err error
+}
+
+// ErrLogStreamStalled indicates that a log stream stopped sending data and may be unresponsive.
+var ErrLogStreamStalled = errors.New("log stream stopped responding")

pkg/api/service.go

@@ -8,6 +8,7 @@ import (
 	"slices"
 	"strings"
 
+	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/distribution/reference"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -366,6 +367,16 @@ type MachineServiceContainer struct {
 	Container ServiceContainer
 }
 
+// MachineIDs returns a list of unique machine IDs where the service containers are running.
+func (s *Service) MachineIDs() []string {
+	ids := mapset.NewSet[string]()
+	for _, mc := range s.Containers {
+		ids.Add(mc.MachineID)
+	}
+
+	return ids.ToSlice()
+}
+
 // Images returns a sorted list of unique images used by the service containers.
 func (s *Service) Images() []string {
 	images := make(map[string]struct{})

pkg/client/client.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/docker/cli/cli/streams"
 	"github.com/psviderski/uncloud/internal/machine/api/pb"
@@ -76,3 +77,41 @@ func proxyToMachine(ctx context.Context, machine *pb.MachineInfo) context.Contex
 	md := metadata.Pairs("machines", machineIP.String())
 	return metadata.NewOutgoingContext(ctx, md)
 }
+
+// ProxyMachinesContext returns a new context that proxies gRPC requests to the specified machines.
+// If namesOrIDs is nil, all machines are included.
+func (cli *Client) ProxyMachinesContext(
+	ctx context.Context, namesOrIDs []string,
+) (context.Context, api.MachineMembersList, error) {
+	// TODO: move the machine IP resolution to the proxy router to allow setting machine names and IDs in the metadata.
+	machines, err := cli.ListMachines(ctx, nil)
+	if err != nil {
+		return nil, nil, fmt.Errorf("list machines: %w", err)
+	}
+
+	var proxiedMachines api.MachineMembersList
+	var notFound []string
+	for _, nameOrID := range namesOrIDs {
+		if m := machines.FindByNameOrID(nameOrID); m != nil {
+			proxiedMachines = append(proxiedMachines, m)
+		} else {
+			notFound = append(notFound, nameOrID)
+		}
+	}
+
+	if len(notFound) > 0 {
+		return nil, nil, fmt.Errorf("machines not found: %s", strings.Join(notFound, ", "))
+	}
+
+	if len(namesOrIDs) == 0 {
+		proxiedMachines = machines
+	}
+
+	md := metadata.New(nil)
+	for _, m := range proxiedMachines {
+		machineIP, _ := m.Machine.Network.ManagementIp.ToAddr()
+		md.Append("machines", machineIP.String())
+	}
+
+	return metadata.NewOutgoingContext(ctx, md), proxiedMachines, nil
+}

pkg/client/image.go

@@ -48,7 +48,7 @@ func (cli *Client) InspectRemoteImage(ctx context.Context, id string) ([]api.Mac
 // it lists images on all machines.
 func (cli *Client) ListImages(ctx context.Context, filter api.ImageFilter) ([]api.MachineImages, error) {
 	// Broadcast the image list request to the specified machines or all machines if none specified.
-	listCtx, machines, err := api.ProxyMachinesContext(ctx, cli, filter.Machines)
+	listCtx, machines, err := cli.ProxyMachinesContext(ctx, filter.Machines)
 	if err != nil {
 		return nil, fmt.Errorf("create request context to broadcast to machines: %w", err)
 	}

pkg/client/logmerger.go

@@ -0,0 +1,310 @@
+package client
+
+import (
+	"container/heap"
+	"sync"
+	"time"
+
+	"github.com/psviderski/uncloud/pkg/api"
+)
+
+// logMergerMaxInFlightPerStream limits how many entries each input stream can have in the processing queue before being
+// throttled. This ensures fair interleaving between streams and prevents one fast stream from causing unbounded
+// buffering while waiting for slower streams.
+const (
+	logMergerMaxInFlightPerStream = 100
+	// logMergerHeartbeatDebounceInterval defines the minimum interval between deduplicated heartbeat entries emitted
+	// by the LogMerger. Keep it in sync with logsHeartbeatInterval internal/machine/docker/server.go.
+	logMergerHeartbeatDebounceInterval = 200 * time.Millisecond
+)
+
+// LogMergerOptions configures the behavior of LogMerger.
+type LogMergerOptions struct {
+	// StallTimeout specifies how long a stream can go without receiving any data before it's considered
+	// stalled and excluded from watermark calculation. A zero timeout disables stall detection.
+	StallTimeout time.Duration
+	// StallCheckInterval specifies how often to check for stalled streams.
+	StallCheckInterval time.Duration
+}
+
+// DefaultLogMergerOptions provides sensible default options that enable stall detection for LogMerger.
+var DefaultLogMergerOptions = LogMergerOptions{
+	StallTimeout:       10 * time.Second,
+	StallCheckInterval: 1 * time.Second,
+}
+
+// LogMerger merges multiple log streams into a single chronologically ordered stream based on timestamps.
+// It uses a low watermark algorithm to ensure proper ordering across streams.
+// Heartbeat entries from streams advance the watermark to enable timely emission of buffered logs.
+type LogMerger struct {
+	streams []*mergerStream
+	queue   logsHeap
+	// watermark is min(latest_timestamp for each stream).
+	watermark time.Time
+	output    chan api.ServiceLogEntry
+	// lastEmitted is the timestamp of the last emitted log entry or heartbeat.
+	lastEmitted        time.Time
+	stallTimeout       time.Duration
+	stallCheckInterval time.Duration
+}
+
+// NewLogMerger creates a new LogMerger for the given input streams with the specified options.
+func NewLogMerger(streams []<-chan api.ServiceLogEntry, opts LogMergerOptions) *LogMerger {
+	mergerStreams := make([]*mergerStream, len(streams))
+	now := time.Now()
+	for i, ch := range streams {
+		mergerStreams[i] = &mergerStream{
+			stream:       ch,
+			semaphore:    make(chan struct{}, logMergerMaxInFlightPerStream),
+			lastActivity: now,
+		}
+	}
+
+	return &LogMerger{
+		streams:            mergerStreams,
+		output:             make(chan api.ServiceLogEntry),
+		stallTimeout:       opts.StallTimeout,
+		stallCheckInterval: opts.StallCheckInterval,
+	}
+}
+
+// Stream starts the merge process and returns a channel that emits log entries in chronological order.
+// The returned channel is closed when all input streams are closed.
+func (m *LogMerger) Stream() <-chan api.ServiceLogEntry {
+	if len(m.streams) == 0 {
+		close(m.output)
+		return m.output
+	}
+
+	go m.run()
+
+	return m.output
+}
+
+// mergerStream combines a stream channel with its state and flow control.
+type mergerStream struct {
+	stream    <-chan api.ServiceLogEntry
+	semaphore chan struct{}
+	// Latest timestamp seen from this stream (log or heartbeat).
+	lastSeen time.Time
+	// Wall clock time when we last received any data from this stream.
+	lastActivity time.Time
+	// Metadata associated with this stream. It's populated from the first log entry received.
+	metadata *api.ServiceLogEntryMetadata
+	// Whether the stream channel has closed.
+	closed bool
+	// Whether the stream is considered stalled (no data received within timeout).
+	stalled bool
+}
+
+// streamEvent represents an event from a stream (entry received or stream closed).
+type streamEvent struct {
+	stream *mergerStream
+	entry  api.ServiceLogEntry
+	closed bool
+}
+
+// queuedEntry wraps a log entry with its source semaphore for release tracking.
+type queuedEntry struct {
+	entry     api.ServiceLogEntry
+	semaphore chan struct{}
+}
+
+// run is the main processing loop that merges all streams.
+func (m *LogMerger) run() {
+	defer close(m.output)
+
+	// Fan-in channel for stream events.
+	events := make(chan streamEvent)
+
+	// Start a reader goroutine for each stream to send entries to the events channel with flow control.
+	var wg sync.WaitGroup
+	for _, stream := range m.streams {
+		wg.Go(func() {
+			for entry := range stream.stream {
+				// Acquire semaphore slot before sending the entry to limit in-flight unprocessed entries per stream.
+				stream.semaphore <- struct{}{}
+				events <- streamEvent{stream: stream, entry: entry}
+			}
+
+			events <- streamEvent{stream: stream, closed: true}
+		})
+	}
+
+	// Close events channel when all readers finish.
+	go func() {
+		wg.Wait()
+		close(events)
+	}()
+
+	// Set up stall detection timer if enabled.
+	var stallCh <-chan time.Time
+	if m.stallTimeout > 0 && m.stallCheckInterval > 0 {
+		stallTicker := time.NewTicker(m.stallCheckInterval)
+		stallCh = stallTicker.C
+		defer stallTicker.Stop()
+	}
+
+	// Process events and emit entries.
+	for {
+		select {
+		case e, ok := <-events:
+			if !ok {
+				// All streams closed: flush remaining entries in order.
+				for m.queue.Len() > 0 {
+					qe := heap.Pop(&m.queue).(queuedEntry)
+					m.output <- qe.entry
+					<-qe.semaphore
+				}
+				return
+			}
+
+			e.stream.lastActivity = time.Now()
+			if e.stream.stalled {
+				e.stream.stalled = false
+			}
+			if e.stream.metadata == nil {
+				e.stream.metadata = &e.entry.Metadata
+			}
+
+			if e.closed {
+				e.stream.closed = true
+
+				m.updateWatermark()
+				m.emitReadyEntries()
+				continue
+			}
+
+			// Forward errors immediately and release semaphore.
+			if e.entry.Err != nil {
+				m.output <- e.entry
+				<-e.stream.semaphore
+				continue
+			}
+
+			if e.entry.Timestamp.After(e.stream.lastSeen) {
+				e.stream.lastSeen = e.entry.Timestamp
+			}
+			if e.entry.Stream == api.LogStreamStdout || e.entry.Stream == api.LogStreamStderr {
+				heap.Push(&m.queue, queuedEntry{entry: e.entry, semaphore: e.stream.semaphore})
+			}
+
+			m.updateWatermark()
+			m.emitReadyEntries()
+
+			// When merging streams, each input emits its own heartbeats. We want to debounce them and emit our own
+			// heartbeats at the same rate as a single input stream. Note that we need to adjust the heartbeat timestamp
+			// to the current watermark to not violate ordering guarantees.
+			if e.entry.Stream == api.LogStreamHeartbeat {
+				if m.watermark.Sub(m.lastEmitted) >= logMergerHeartbeatDebounceInterval {
+					heartbeat := e.entry
+					heartbeat.Timestamp = m.watermark
+					m.output <- heartbeat
+					m.lastEmitted = m.watermark
+				}
+				// Heartbeat processed, release semaphore.
+				<-e.stream.semaphore
+			}
+
+		case <-stallCh:
+			stalled := m.checkStalledStreams()
+			if len(stalled) == 0 {
+				continue
+			}
+
+			for _, s := range stalled {
+				errEntry := api.ServiceLogEntry{
+					ContainerLogEntry: api.ContainerLogEntry{
+						Err: api.ErrLogStreamStalled,
+					},
+				}
+				if s.metadata != nil {
+					errEntry.Metadata = *s.metadata
+				}
+
+				m.output <- errEntry
+			}
+
+			m.updateWatermark()
+			m.emitReadyEntries()
+		}
+	}
+}
+
+// checkStalledStreams marks streams as stalled if they haven't received any data within the timeout.
+// Returns true if any stream's stalled state changed.
+func (m *LogMerger) checkStalledStreams() []*mergerStream {
+	var stalled []*mergerStream
+	now := time.Now()
+
+	for _, s := range m.streams {
+		if s.closed || s.stalled {
+			continue
+		}
+
+		if now.Sub(s.lastActivity) > m.stallTimeout {
+			s.stalled = true
+			stalled = append(stalled, s)
+		}
+	}
+
+	return stalled
+}
+
+// updateWatermark recalculates the low watermark based on the lastSeen timestamps of all active streams.
+func (m *LogMerger) updateWatermark() {
+	first := true
+
+	for _, s := range m.streams {
+		if s.closed || s.stalled {
+			// Closed and stalled streams don't affect watermark.
+			continue
+		}
+		if first || s.lastSeen.Before(m.watermark) {
+			m.watermark = s.lastSeen
+			first = false
+		}
+	}
+}
+
+// emitReadyEntries pops and emits all buffered entries from the queue with timestamp before the watermark.
+func (m *LogMerger) emitReadyEntries() {
+	if m.watermark.IsZero() {
+		// No entries received yet.
+		return
+	}
+
+	for m.queue.Len() > 0 && m.queue[0].entry.Timestamp.Compare(m.watermark) <= 0 {
+		qe := heap.Pop(&m.queue).(queuedEntry)
+		m.output <- qe.entry
+		m.lastEmitted = qe.entry.Timestamp
+		<-qe.semaphore
+	}
+}
+
+// logsHeap is a min-heap (heap.Interface) of queued entries ordered by timestamp.
+type logsHeap []queuedEntry
+
+func (h *logsHeap) Len() int {
+	return len(*h)
+}
+
+func (h *logsHeap) Less(i, j int) bool {
+	return (*h)[i].entry.Timestamp.Before((*h)[j].entry.Timestamp)
+}
+
+func (h *logsHeap) Swap(i, j int) {
+	(*h)[i], (*h)[j] = (*h)[j], (*h)[i]
+}
+
+func (h *logsHeap) Push(x any) {
+	*h = append(*h, x.(queuedEntry))
+}
+
+func (h *logsHeap) Pop() any {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[:n-1]
+	return x
+}

pkg/client/logmerger_test.go

@@ -0,0 +1,383 @@
+package client
+
+import (
+	"testing"
+	"time"
+
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testEntry creates a ServiceLogEntry for testing.
+func testEntry(stream api.LogStreamType, ts time.Time, msg string) api.ServiceLogEntry {
+	return api.ServiceLogEntry{
+		ContainerLogEntry: api.ContainerLogEntry{
+			Stream:    stream,
+			Timestamp: ts,
+			Message:   []byte(msg),
+		},
+	}
+}
+
+// collectEntries collects up to maxCount entries from the channel or until it is closed.
+func collectEntries(t *testing.T, ch <-chan api.ServiceLogEntry, maxCount int) []api.ServiceLogEntry {
+	t.Helper()
+	var entries []api.ServiceLogEntry
+
+	for i := 0; i < maxCount || maxCount <= 0; i++ {
+		select {
+		case e, ok := <-ch:
+			if !ok {
+				return entries
+			}
+			entries = append(entries, e)
+		case <-time.After(1 * time.Second):
+			require.FailNow(t, "timed out waiting for log entry")
+		}
+	}
+
+	return entries
+}
+
+func TestLogMerger_EmptyStreams(t *testing.T) {
+	t.Parallel()
+
+	merger := NewLogMerger(nil, LogMergerOptions{})
+	output := merger.Stream()
+
+	_, ok := <-output
+	assert.False(t, ok, "output channel should be closed for empty streams")
+}
+
+func TestLogMerger_SingleStream(t *testing.T) {
+	t.Parallel()
+
+	ch := make(chan api.ServiceLogEntry, 10)
+	merger := NewLogMerger([]<-chan api.ServiceLogEntry{ch}, LogMergerOptions{})
+	output := merger.Stream()
+
+	t1 := time.Now()
+	t2 := t1.Add(time.Second)
+
+	ch <- testEntry(api.LogStreamStdout, t1, "first")
+	ch <- testEntry(api.LogStreamStdout, t2, "second")
+
+	results := collectEntries(t, output, 2)
+
+	require.Len(t, results, 2)
+	assert.Equal(t, "first", string(results[0].Message))
+	assert.Equal(t, "second", string(results[1].Message))
+
+	close(ch)
+
+	results = collectEntries(t, output, 0)
+	assert.Len(t, results, 0, "no entries expected after close")
+}
+
+func TestLogMerger_PreservesData(t *testing.T) {
+	t.Parallel()
+
+	ch := make(chan api.ServiceLogEntry, 10)
+	merger := NewLogMerger([]<-chan api.ServiceLogEntry{ch}, LogMergerOptions{})
+	output := merger.Stream()
+
+	metadata := api.ServiceLogEntryMetadata{
+		ServiceID:   "svc-123",
+		ServiceName: "my-service",
+		MachineID:   "machine-456",
+		MachineName: "machine-1",
+	}
+
+	e := api.ServiceLogEntry{
+		Metadata: metadata,
+		ContainerLogEntry: api.ContainerLogEntry{
+			Stream:    api.LogStreamStdout,
+			Timestamp: time.Now(),
+			Message:   []byte("test"),
+		},
+	}
+	ch <- e
+	close(ch)
+
+	results := collectEntries(t, output, 0)
+	require.Len(t, results, 1)
+	assert.Equal(t, e, results[0])
+}
+
+func TestLogMerger_BasicMerge(t *testing.T) {
+	t.Parallel()
+
+	ch1 := make(chan api.ServiceLogEntry, 10)
+	ch2 := make(chan api.ServiceLogEntry, 10)
+
+	merger := NewLogMerger([]<-chan api.ServiceLogEntry{ch1, ch2}, LogMergerOptions{})
+	output := merger.Stream()
+
+	t1 := time.Now()
+	t2 := t1.Add(time.Second)
+	t3 := t1.Add(2 * time.Second)
+	t4 := t1.Add(3 * time.Second)
+
+	// Send interleaved entries from both streams.
+	ch1 <- testEntry(api.LogStreamStdout, t1, "ch1-first")
+	ch2 <- testEntry(api.LogStreamStdout, t2, "ch2-first")
+	ch1 <- testEntry(api.LogStreamStdout, t3, "ch1-second")
+	ch2 <- testEntry(api.LogStreamStdout, t4, "ch2-second")
+
+	// Send heartbeats to advance watermark past all entries.
+	t5 := t1.Add(4 * time.Second)
+	ch1 <- testEntry(api.LogStreamHeartbeat, t5, "")
+	ch2 <- testEntry(api.LogStreamHeartbeat, t5, "")
+
+	results := collectEntries(t, output, 4)
+	require.Len(t, results, 4)
+	assert.Equal(t, "ch1-first", string(results[0].Message))
+	assert.Equal(t, "ch2-first", string(results[1].Message))
+	assert.Equal(t, "ch1-second", string(results[2].Message))
+	assert.Equal(t, "ch2-second", string(results[3].Message))
+
+	close(ch1)
+	close(ch2)
+
+	results = collectEntries(t, output, 0)
+	assert.Len(t, results, 1, "one debounced heartbeat expected after close")
+	assert.Equal(t, api.LogStreamHeartbeat, results[0].Stream)
+	assert.Equal(t, t5, results[0].Timestamp)
+}
+
+func TestLogMerger_HeartbeatAdvancesWatermark(t *testing.T) {
+	t.Parallel()
+
+	ch1 := make(chan api.ServiceLogEntry, 10)
+	ch2 := make(chan api.ServiceLogEntry, 10)
+
+	merger := NewLogMerger([]<-chan api.ServiceLogEntry{ch1, ch2}, LogMergerOptions{})
+	output := merger.Stream()
+
+	t1 := time.Now()
+	t2 := t1.Add(time.Second)
+	t3 := t1.Add(2 * time.Second)
+
+	// Stream 1 sends a log entry.
+	ch1 <- testEntry(api.LogStreamStdout, t1, "ch1-first")
+	ch1 <- testEntry(api.LogStreamStdout, t3, "ch1-second")
+	// Stream 2 is quiet but sends a heartbeat.
+	ch2 <- testEntry(api.LogStreamHeartbeat, t2, "")
+
+	// Now stream 1's entry should be emitted because watermark is t1 (min of t1, t2).
+	results := collectEntries(t, output, 1)
+	require.Len(t, results, 1)
+	assert.Equal(t, "ch1-first", string(results[0].Message))
+
+	close(ch1)
+	close(ch2)
+
+	results = collectEntries(t, output, 0)
+	require.NotEmpty(t, results)
+
+	// Depending on timing, we may get a heartbeat before the second log entry.
+	require.LessOrEqual(t, len(results), 2, "one or two entries expected after close")
+	if len(results) == 2 {
+		assert.Equal(t, api.LogStreamHeartbeat, results[0].Stream)
+		assert.Equal(t, t2, results[0].Timestamp)
+		results = results[1:]
+	}
+	assert.Equal(t, "ch1-second", string(results[0].Message))
+}
+
+func TestLogMerger_ErrorForwarding(t *testing.T) {
+	t.Parallel()
+
+	ch1 := make(chan api.ServiceLogEntry, 10)
+	ch2 := make(chan api.ServiceLogEntry, 10)
+	merger := NewLogMerger([]<-chan api.ServiceLogEntry{ch1, ch2}, LogMergerOptions{})
+	output := merger.Stream()
+
+	t1 := time.Now()
+
+	ch1 <- testEntry(api.LogStreamStdout, t1, "ch1-first")
+	// Send an error entry.
+	ch1 <- api.ServiceLogEntry{
+		ContainerLogEntry: api.ContainerLogEntry{
+			Err: assert.AnError,
+		},
+	}
+
+	results := collectEntries(t, output, 1)
+	require.Len(t, results, 1, "expected error to emitted out of order")
+	assert.Equal(t, assert.AnError, results[0].Err)
+
+	close(ch1)
+	close(ch2)
+
+	results = collectEntries(t, output, 0)
+	assert.Len(t, results, 1)
+	assert.Equal(t, "ch1-first", string(results[0].Message))
+}
+
+func TestLogMerger_OutOfOrderSingleStream(t *testing.T) {
+	t.Parallel()
+
+	ch := make(chan api.ServiceLogEntry, 10)
+	merger := NewLogMerger([]<-chan api.ServiceLogEntry{ch}, LogMergerOptions{})
+	output := merger.Stream()
+
+	t1 := time.Now()
+	t2 := t1.Add(time.Second)
+	t3 := t1.Add(2 * time.Second)
+	t4 := t1.Add(3 * time.Second)
+
+	// Send entries out of order within the stream.
+	ch <- testEntry(api.LogStreamStdout, t3, "third")
+	ch <- testEntry(api.LogStreamStdout, t2, "second")
+	ch <- testEntry(api.LogStreamStdout, t4, "forth")
+	ch <- testEntry(api.LogStreamStdout, t1, "first")
+
+	results := collectEntries(t, output, 4)
+	require.Len(t, results, 4)
+	// Emitted in the same order as sent since no buffering/reordering is done within a single stream.
+	assert.Equal(t, "third", string(results[0].Message))
+	assert.Equal(t, "second", string(results[1].Message))
+	assert.Equal(t, "forth", string(results[2].Message))
+	assert.Equal(t, "first", string(results[3].Message))
+
+	close(ch)
+
+	results = collectEntries(t, output, 0)
+	assert.Len(t, results, 0, "no entries expected after close")
+}
+
+// Test that entries from streams with different rates are merged correctly in chronological order.
+func TestLogMerger_UnevenStreams(t *testing.T) {
+	t.Parallel()
+
+	numFastEntries := logMergerMaxInFlightPerStream
+
+	// Use buffered channels to allow sends without blocking on receiver.
+	ch1 := make(chan api.ServiceLogEntry, numFastEntries+5)
+	ch2 := make(chan api.ServiceLogEntry, 10)
+
+	merger := NewLogMerger([]<-chan api.ServiceLogEntry{ch1, ch2}, LogMergerOptions{})
+	output := merger.Stream()
+
+	baseTime := time.Now()
+
+	// Stream 1 sends many entries quickly (0ms - 104ms).
+	for i := 0; i < numFastEntries; i++ {
+		ch1 <- testEntry(api.LogStreamStdout, baseTime.Add(time.Duration(i)*time.Millisecond), "fast")
+	}
+
+	// Stream 2 sends entries that interleave with fast entries.
+	ch2 <- testEntry(api.LogStreamStdout, baseTime.Add(5*time.Millisecond), "slow")
+	ch2 <- testEntry(api.LogStreamStdout, baseTime.Add(50*time.Millisecond), "slow")
+
+	// Send heartbeats to advance watermark past all entries so they get emitted.
+	finalTime := baseTime.Add(time.Second)
+	ch1 <- testEntry(api.LogStreamHeartbeat, finalTime, "")
+	ch2 <- testEntry(api.LogStreamHeartbeat, finalTime.Add(10*time.Millisecond), "")
+
+	// Read exactly numFastEntries + 2 log entries.
+	expectedCount := numFastEntries + 2
+	results := collectEntries(t, output, expectedCount)
+	require.Len(t, results, expectedCount)
+
+	// Count entries from each stream.
+	fastCount := 0
+	slowCount := 0
+	for _, entry := range results {
+		switch string(entry.Message) {
+		case "fast":
+			fastCount++
+		case "slow":
+			slowCount++
+		}
+	}
+	assert.Equal(t, numFastEntries, fastCount)
+	assert.Equal(t, 2, slowCount)
+
+	// Verify entries are in chronological order.
+	for i := 1; i < len(results); i++ {
+		assert.False(t, results[i].Timestamp.Before(results[i-1].Timestamp),
+			"entry %d (ts=%v) should not be before entry %d (ts=%v)",
+			i, results[i].Timestamp, i-1, results[i-1].Timestamp)
+	}
+
+	close(ch1)
+	close(ch2)
+
+	results = collectEntries(t, output, 0)
+	require.Len(t, results, 1, "one debounced heartbeat expected after close")
+	assert.Equal(t, api.LogStreamHeartbeat, results[0].Stream)
+}
+
+func TestLogMerger_StalledStreamExcludedFromWatermark(t *testing.T) {
+	t.Parallel()
+
+	ch1 := make(chan api.ServiceLogEntry, 10)
+	ch2 := make(chan api.ServiceLogEntry, 10)
+
+	stallTimeout := 100 * time.Millisecond
+	merger := NewLogMerger([]<-chan api.ServiceLogEntry{ch1, ch2}, LogMergerOptions{
+		StallTimeout:       stallTimeout,
+		StallCheckInterval: 20 * time.Millisecond,
+	})
+	output := merger.Stream()
+
+	//// Collect all output entries in a separate goroutine.
+	//var results []api.ServiceLogEntry
+	//doneCollecting := make(chan struct{})
+	//go func() {
+	//	for entry := range output {
+	//		results = append(results, entry)
+	//	}
+	//	close(doneCollecting)
+	//}()
+
+	t1 := time.Now()
+	t2 := t1.Add(time.Second)
+	t3 := t1.Add(2 * time.Second)
+	t4 := t1.Add(3 * time.Second)
+
+	e1 := testEntry(api.LogStreamStdout, t1, "ch1-entry")
+	e1.Metadata = api.ServiceLogEntryMetadata{
+		ServiceID:   "ch1-svc1",
+		ServiceName: "ch1-svcName",
+		ContainerID: "ch1-ctr1",
+		MachineID:   "ch1-machine1",
+		MachineName: "ch1-machineName",
+	}
+
+	ch1 <- e1
+	ch2 <- testEntry(api.LogStreamStdout, t2, "ch2-first")
+
+	// Watermark is t1 now so ch1's entry could be collected.
+	results := collectEntries(t, output, 1)
+	require.Len(t, results, 1)
+	assert.Equal(t, "ch1-entry", string(results[0].Message))
+
+	// Keep pushing to stream 2 so that it does not stall. But it can't emit anything because watermark is stuck at t1.
+	time.Sleep(stallTimeout / 2)
+	ch2 <- testEntry(api.LogStreamStdout, t3, "ch2-second")
+
+	// 1/2 + 2/3 = 7/6 > 1, so stream 1 should be considered stalled now.
+	time.Sleep(stallTimeout / 3 * 2)
+	ch2 <- testEntry(api.LogStreamStdout, t4, "ch2-third")
+
+	results = collectEntries(t, output, 1)
+	require.ErrorIs(t, results[0].Err, api.ErrLogStreamStalled)
+	assert.Equal(t, e1.Metadata, results[0].Metadata, "stalled entry should have stream metadata")
+
+	// Now that stream 1 is marked as stalled and ignored, all in-flight entries from stream 2 are emitted.
+	results = collectEntries(t, output, 3)
+	require.Len(t, results, 3)
+	assert.Equal(t, "ch2-first", string(results[0].Message))
+	assert.Equal(t, "ch2-second", string(results[1].Message))
+	assert.Equal(t, "ch2-third", string(results[2].Message))
+
+	close(ch1)
+	close(ch2)
+
+	results = collectEntries(t, output, 0)
+	assert.Len(t, results, 0)
+}

pkg/client/logs.go

@@ -0,0 +1,141 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/psviderski/uncloud/internal/machine/api/pb"
+	"github.com/psviderski/uncloud/pkg/api"
+)
+
+// ServiceLogs streams log entries from all service containers in chronological order based on timestamps.
+// Keep in mind that perfect ordering of log events across multiple machines can't be guaranteed due to the
+// imperfection of physical clocks or potential clock skew between machines.
+// It uses a low watermark algorithm to ensure proper ordering across multiple machines.
+// Heartbeat entries from the server advance the watermark to enable timely emission of buffered logs.
+func (cli *Client) ServiceLogs(
+	ctx context.Context, serviceNameOrID string, opts api.ServiceLogsOptions,
+) (api.Service, <-chan api.ServiceLogEntry, error) {
+	svc, err := cli.InspectService(ctx, serviceNameOrID)
+	if err != nil {
+		return svc, nil, fmt.Errorf("inspect service: %w", err)
+	}
+
+	if len(svc.Containers) == 0 {
+		return svc, nil, fmt.Errorf("no containers found for service: %s", serviceNameOrID)
+	}
+
+	machines, err := cli.ListMachines(ctx, nil)
+	if err != nil {
+		return svc, nil, fmt.Errorf("list machines: %w", err)
+	}
+
+	ctrStreams := make([]<-chan api.ServiceLogEntry, 0, len(svc.Containers))
+	for _, ctr := range svc.Containers {
+		// Try to get machine name for ServiceLogEntry metadata and friendlier error message.
+		machineName := ctr.MachineID
+		m := machines.FindByNameOrID(ctr.MachineID)
+		if m != nil {
+			machineName = m.Machine.Name
+		}
+
+		stream, err := cli.ContainerLogs(ctx, ctr.MachineID, ctr.Container.ID, opts)
+		if err != nil {
+			return svc, nil, fmt.Errorf("stream logs from service container '%s' on machine '%s': %w",
+				stringid.TruncateID(ctr.Container.ID), machineName, err)
+		}
+
+		// Enrich log entries from the container with service metadata.
+		metadata := api.ServiceLogEntryMetadata{
+			ServiceID:   svc.ID,
+			ServiceName: svc.Name,
+			ContainerID: ctr.Container.ID,
+			MachineID:   ctr.MachineID,
+			MachineName: machineName,
+		}
+		enrichedStream := logsStreamWithServiceMetadata(stream, metadata)
+		ctrStreams = append(ctrStreams, enrichedStream)
+	}
+
+	// Use the log merger to combine streams from all containers in chronological order.
+	merger := NewLogMerger(ctrStreams, DefaultLogMergerOptions)
+	mergedStream := merger.Stream()
+
+	return svc, mergedStream, nil
+}
+
+// ContainerLogs streams log entries from a single container on a specified machine.
+func (cli *Client) ContainerLogs(
+	ctx context.Context, machineNameOrID string, containerID string, opts api.ServiceLogsOptions,
+) (<-chan api.ContainerLogEntry, error) {
+	proxyCtx, _, err := cli.ProxyMachinesContext(ctx, []string{machineNameOrID})
+	if err != nil {
+		return nil, fmt.Errorf("create request context to proxy to machine '%s': %w", machineNameOrID, err)
+	}
+
+	req := &pb.ContainerLogsRequest{
+		ContainerId: containerID,
+		Follow:      opts.Follow,
+		Tail:        int32(opts.Tail),
+		Since:       opts.Since,
+		Until:       opts.Until,
+	}
+	stream, err := cli.Docker.GRPCClient.ContainerLogs(proxyCtx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	ch := make(chan api.ContainerLogEntry)
+
+	go func() {
+		defer close(ch)
+
+		for {
+			pbEntry, err := stream.Recv()
+			if err == io.EOF {
+				return
+			}
+			if err != nil {
+				ch <- api.ContainerLogEntry{
+					Err: err,
+				}
+				return
+			}
+
+			entry := api.ContainerLogEntry{
+				Stream:    api.LogStreamTypeFromProto(pbEntry.Stream),
+				Message:   pbEntry.Message,
+				Timestamp: pbEntry.Timestamp.AsTime(),
+			}
+
+			select {
+			case ch <- entry:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return ch, nil
+}
+
+// logsStreamWithServiceMetadata wraps a container logs stream and enriches each log entry with service metadata.
+func logsStreamWithServiceMetadata(
+	stream <-chan api.ContainerLogEntry, metadata api.ServiceLogEntryMetadata,
+) <-chan api.ServiceLogEntry {
+	out := make(chan api.ServiceLogEntry)
+
+	go func() {
+		for entry := range stream {
+			out <- api.ServiceLogEntry{
+				Metadata:          metadata,
+				ContainerLogEntry: entry,
+			}
+		}
+		close(out)
+	}()
+
+	return out
+}

pkg/client/machine.go

@@ -48,6 +48,26 @@ func (cli *Client) ListMachines(ctx context.Context, filter *api.MachineFilter)
 	return machines, nil
 }
 
+func MachineMatchesFilter(machine *pb.MachineMember, filter *api.MachineFilter) bool {
+	if filter == nil {
+		return true
+	}
+
+	if filter.Available && machine.State == pb.MachineMember_DOWN {
+		return false
+	}
+
+	if len(filter.NamesOrIDs) > 0 {
+		if !slices.ContainsFunc(filter.NamesOrIDs, func(nameOrID string) bool {
+			return machine.Machine.Id == nameOrID || machine.Machine.Name == nameOrID
+		}) {
+			return false
+		}
+	}
+
+	return true
+}
+
 // UpdateMachine updates machine configuration in the cluster.
 func (cli *Client) UpdateMachine(ctx context.Context, req *pb.UpdateMachineRequest) (*pb.MachineInfo, error) {
 	resp, err := cli.ClusterClient.UpdateMachine(ctx, req)
@@ -76,23 +96,3 @@ func (cli *Client) RenameMachine(ctx context.Context, nameOrID, newName string)
 
 	return cli.UpdateMachine(ctx, req)
 }
-
-func MachineMatchesFilter(machine *pb.MachineMember, filter *api.MachineFilter) bool {
-	if filter == nil {
-		return true
-	}
-
-	if filter.Available && machine.State == pb.MachineMember_DOWN {
-		return false
-	}
-
-	if len(filter.NamesOrIDs) > 0 {
-		if !slices.ContainsFunc(filter.NamesOrIDs, func(nameOrID string) bool {
-			return machine.Machine.Id == nameOrID || machine.Machine.Name == nameOrID
-		}) {
-			return false
-		}
-	}
-
-	return true
-}

pkg/client/volume.go

@@ -55,7 +55,7 @@ func (cli *Client) ListVolumes(ctx context.Context, filter *api.VolumeFilter) ([
 		proxyMachines = filter.Machines
 	}
 
-	listCtx, machines, err := api.ProxyMachinesContext(ctx, cli, proxyMachines)
+	listCtx, machines, err := cli.ProxyMachinesContext(ctx, proxyMachines)
 	if err != nil {
 		return nil, fmt.Errorf("create request context to broadcast to all machines: %w", err)
 	}