feat: service logs command to stream service logs (#196)

* chore: fix landing logo shadow

* feat: implement service logs command with colored output and strict ordering

* revert Makefile

* fix after rebase

* move logs command under services with root shortcut

* simplify server options for streaming logs and CLI flags

* refactor ContainerLogs grpc server

* update --tail flag

* minor docker.proto

* refactor client ServiceLogs and ContainerLogs

* move ProxyMachinesContext from api to client pkg

* minor refactor proto Stream

* add api/logs

* implement LogMerger

* fix LogMerger to correctly use semaphore

* increate inflish entries to 100 per stream

* send heartbeats

* refactor ContainerLogs to synchronise Send of entries and heartbeats to the steram

* minor logmerege

* remove ContainerName form ServiceLogEntryMetadata

* refactor ContainerLogs into docker.Service

* detect stalled container logs streams

* update logmerger tests

* fix comment in test

* refactor LogMerger with options

* uc logs: format one or multiple services

* make LogMerger emit heartbeats, emit entries <= watermark, rewrite tests

* update uc logs with new LogMerger

* go mod tidy

* fix after merge

---------

Co-authored-by: Evgenii Orlov <evgenii.orlov@semrush.com>

internal/machine/api/pb/docker.proto

@@ -5,6 +5,7 @@ package api;
 option go_package = "github.com/psviderski/uncloud/internal/machine/api/pb";
 
 import "google/protobuf/empty.proto";
+import "google/protobuf/timestamp.proto";
 import "internal/machine/api/pb/common.proto";
 
 service Docker {
@@ -15,6 +16,9 @@ service Docker {
   rpc ListContainers(ListContainersRequest) returns (ListContainersResponse);
   rpc RemoveContainer(RemoveContainerRequest) returns (google.protobuf.Empty);
 
+  rpc ExecContainer(stream ExecContainerRequest) returns (stream ExecContainerResponse);
+  rpc ContainerLogs(ContainerLogsRequest) returns (stream ContainerLogEntry);
+
   rpc PullImage(PullImageRequest) returns (stream JSONMessage);
   rpc InspectImage(InspectImageRequest) returns (InspectImageResponse);
   // InspectRemoteImage returns the image metadata for an image in a remote registry using the machine's
@@ -30,8 +34,6 @@ service Docker {
   rpc InspectServiceContainer(InspectContainerRequest) returns (ServiceContainer);
   rpc ListServiceContainers(ListServiceContainersRequest) returns (ListServiceContainersResponse);
   rpc RemoveServiceContainer(RemoveContainerRequest) returns (google.protobuf.Empty);
-
-  rpc ExecContainer(stream ExecContainerRequest) returns (stream ExecContainerResponse);
 }
 
 message CreateContainerRequest {
@@ -94,6 +96,64 @@ message RemoveContainerRequest {
   bytes options = 2;
 }
 
+message ExecContainerRequest {
+  oneof payload {
+    // Initial configuration for the exec session. Must be sent as the first message.
+    ExecConfig config = 1;
+    // Raw stdin data to be written to the exec process.
+    bytes stdin = 2;
+    // TTY resize event (only used when TTY is enabled).
+    ResizeEvent resize = 3;
+  }
+}
+
+message ExecConfig {
+  // Container ID to execute the command in.
+  string container_id = 1;
+  // JSON serialised ExecOptions
+  bytes options = 2;
+}
+
+message ResizeEvent {
+  uint32 height = 1;
+  uint32 width = 2;
+}
+
+message ExecContainerResponse {
+  oneof payload {
+    // Exec instance ID returned after creating the exec.
+    string exec_id = 1;
+    // Raw stdout data from the exec process.
+    bytes stdout = 2;
+    // Raw stderr data from the exec process (only when TTY is disabled).
+    bytes stderr = 3;
+    // Exit code of the exec process. Sent as the final message.
+    int32 exit_code = 4;
+  }
+}
+
+message ContainerLogsRequest {
+  string container_id = 1;
+  // Options for logs retrieval.
+  bool follow = 2;
+  int32 tail = 3; // -1 means all
+  string since = 4; // https://www.rfc-editor.org/rfc/rfc3339.html timestamp or Go duration string
+  string until = 5; // https://www.rfc-editor.org/rfc/rfc3339.html timestamp or Go duration string
+}
+
+message ContainerLogEntry {
+  enum StreamType {
+    UNKNOWN = 0;
+    STDOUT = 1;
+    STDERR = 2;
+    HEARTBEAT = 3;
+  }
+  StreamType stream = 1;
+  google.protobuf.Timestamp timestamp = 2;
+  // Log line content. Empty for heartbeat entries.
+  bytes message = 3;
+}
+
 message PullImageRequest {
   string image = 1;
   // JSON serialised image.PullOptions.
@@ -216,39 +276,3 @@ message MachineServiceContainers {
   Metadata metadata = 1;
   repeated ServiceContainer containers = 2;
 }
-
-message ExecContainerRequest {
-  oneof payload {
-    // Initial configuration for the exec session. Must be sent as the first message.
-    ExecConfig config = 1;
-    // Raw stdin data to be written to the exec process.
-    bytes stdin = 2;
-    // TTY resize event (only used when TTY is enabled).
-    ResizeEvent resize = 3;
-  }
-}
-
-message ExecConfig {
-  // Container ID to execute the command in.
-  string container_id = 1;
-  // JSON serialised ExecOptions
-  bytes options = 2;
-}
-
-message ResizeEvent {
-  uint32 height = 1;
-  uint32 width = 2;
-}
-
-message ExecContainerResponse {
-  oneof payload {
-    // Exec instance ID returned after creating the exec.
-    string exec_id = 1;
-    // Raw stdout data from the exec process.
-    bytes stdout = 2;
-    // Raw stderr data from the exec process (only when TTY is disabled).
-    bytes stderr = 3;
-    // Exit code of the exec process. Sent as the final message.
-    int32 exit_code = 4;
-  }
-}

internal/machine/api/pb/docker_grpc.pb.go

@@ -26,6 +26,8 @@ const (
 	Docker_StopContainer_FullMethodName           = "/api.Docker/StopContainer"
 	Docker_ListContainers_FullMethodName          = "/api.Docker/ListContainers"
 	Docker_RemoveContainer_FullMethodName         = "/api.Docker/RemoveContainer"
+	Docker_ExecContainer_FullMethodName           = "/api.Docker/ExecContainer"
+	Docker_ContainerLogs_FullMethodName           = "/api.Docker/ContainerLogs"
 	Docker_PullImage_FullMethodName               = "/api.Docker/PullImage"
 	Docker_InspectImage_FullMethodName            = "/api.Docker/InspectImage"
 	Docker_InspectRemoteImage_FullMethodName      = "/api.Docker/InspectRemoteImage"
@@ -37,7 +39,6 @@ const (
 	Docker_InspectServiceContainer_FullMethodName = "/api.Docker/InspectServiceContainer"
 	Docker_ListServiceContainers_FullMethodName   = "/api.Docker/ListServiceContainers"
 	Docker_RemoveServiceContainer_FullMethodName  = "/api.Docker/RemoveServiceContainer"
-	Docker_ExecContainer_FullMethodName           = "/api.Docker/ExecContainer"
 )
 
 // DockerClient is the client API for Docker service.
@@ -50,6 +51,8 @@ type DockerClient interface {
 	StopContainer(ctx context.Context, in *StopContainerRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
 	ListContainers(ctx context.Context, in *ListContainersRequest, opts ...grpc.CallOption) (*ListContainersResponse, error)
 	RemoveContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
+	ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error)
+	ContainerLogs(ctx context.Context, in *ContainerLogsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ContainerLogEntry], error)
 	PullImage(ctx context.Context, in *PullImageRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[JSONMessage], error)
 	InspectImage(ctx context.Context, in *InspectImageRequest, opts ...grpc.CallOption) (*InspectImageResponse, error)
 	// InspectRemoteImage returns the image metadata for an image in a remote registry using the machine's
@@ -63,7 +66,6 @@ type DockerClient interface {
 	InspectServiceContainer(ctx context.Context, in *InspectContainerRequest, opts ...grpc.CallOption) (*ServiceContainer, error)
 	ListServiceContainers(ctx context.Context, in *ListServiceContainersRequest, opts ...grpc.CallOption) (*ListServiceContainersResponse, error)
 	RemoveServiceContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
-	ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error)
 }
 
 type dockerClient struct {
@@ -134,9 +136,41 @@ func (c *dockerClient) RemoveContainer(ctx context.Context, in *RemoveContainerR
 	return out, nil
 }
 
+func (c *dockerClient) ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[0], Docker_ExecContainer_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[ExecContainerRequest, ExecContainerResponse]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ExecContainerClient = grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse]
+
+func (c *dockerClient) ContainerLogs(ctx context.Context, in *ContainerLogsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ContainerLogEntry], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[1], Docker_ContainerLogs_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[ContainerLogsRequest, ContainerLogEntry]{ClientStream: stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ContainerLogsClient = grpc.ServerStreamingClient[ContainerLogEntry]
+
 func (c *dockerClient) PullImage(ctx context.Context, in *PullImageRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[JSONMessage], error) {
 	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
-	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[0], Docker_PullImage_FullMethodName, cOpts...)
+	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[2], Docker_PullImage_FullMethodName, cOpts...)
 	if err != nil {
 		return nil, err
 	}
@@ -253,19 +287,6 @@ func (c *dockerClient) RemoveServiceContainer(ctx context.Context, in *RemoveCon
 	return out, nil
 }
 
-func (c *dockerClient) ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error) {
-	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
-	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[1], Docker_ExecContainer_FullMethodName, cOpts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &grpc.GenericClientStream[ExecContainerRequest, ExecContainerResponse]{ClientStream: stream}
-	return x, nil
-}
-
-// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
-type Docker_ExecContainerClient = grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse]
-
 // DockerServer is the server API for Docker service.
 // All implementations must embed UnimplementedDockerServer
 // for forward compatibility.
@@ -276,6 +297,8 @@ type DockerServer interface {
 	StopContainer(context.Context, *StopContainerRequest) (*emptypb.Empty, error)
 	ListContainers(context.Context, *ListContainersRequest) (*ListContainersResponse, error)
 	RemoveContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error)
+	ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error
+	ContainerLogs(*ContainerLogsRequest, grpc.ServerStreamingServer[ContainerLogEntry]) error
 	PullImage(*PullImageRequest, grpc.ServerStreamingServer[JSONMessage]) error
 	InspectImage(context.Context, *InspectImageRequest) (*InspectImageResponse, error)
 	// InspectRemoteImage returns the image metadata for an image in a remote registry using the machine's
@@ -289,7 +312,6 @@ type DockerServer interface {
 	InspectServiceContainer(context.Context, *InspectContainerRequest) (*ServiceContainer, error)
 	ListServiceContainers(context.Context, *ListServiceContainersRequest) (*ListServiceContainersResponse, error)
 	RemoveServiceContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error)
-	ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error
 	mustEmbedUnimplementedDockerServer()
 }
 
@@ -318,6 +340,12 @@ func (UnimplementedDockerServer) ListContainers(context.Context, *ListContainers
 func (UnimplementedDockerServer) RemoveContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method RemoveContainer not implemented")
 }
+func (UnimplementedDockerServer) ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method ExecContainer not implemented")
+}
+func (UnimplementedDockerServer) ContainerLogs(*ContainerLogsRequest, grpc.ServerStreamingServer[ContainerLogEntry]) error {
+	return status.Errorf(codes.Unimplemented, "method ContainerLogs not implemented")
+}
 func (UnimplementedDockerServer) PullImage(*PullImageRequest, grpc.ServerStreamingServer[JSONMessage]) error {
 	return status.Errorf(codes.Unimplemented, "method PullImage not implemented")
 }
@@ -351,9 +379,6 @@ func (UnimplementedDockerServer) ListServiceContainers(context.Context, *ListSer
 func (UnimplementedDockerServer) RemoveServiceContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method RemoveServiceContainer not implemented")
 }
-func (UnimplementedDockerServer) ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error {
-	return status.Errorf(codes.Unimplemented, "method ExecContainer not implemented")
-}
 func (UnimplementedDockerServer) mustEmbedUnimplementedDockerServer() {}
 func (UnimplementedDockerServer) testEmbeddedByValue()                {}
 
@@ -483,6 +508,24 @@ func _Docker_RemoveContainer_Handler(srv interface{}, ctx context.Context, dec f
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Docker_ExecContainer_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(DockerServer).ExecContainer(&grpc.GenericServerStream[ExecContainerRequest, ExecContainerResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ExecContainerServer = grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]
+
+func _Docker_ContainerLogs_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(ContainerLogsRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(DockerServer).ContainerLogs(m, &grpc.GenericServerStream[ContainerLogsRequest, ContainerLogEntry]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ContainerLogsServer = grpc.ServerStreamingServer[ContainerLogEntry]
+
 func _Docker_PullImage_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(PullImageRequest)
 	if err := stream.RecvMsg(m); err != nil {
@@ -674,13 +717,6 @@ func _Docker_RemoveServiceContainer_Handler(srv interface{}, ctx context.Context
 	return interceptor(ctx, in, info, handler)
 }
 
-func _Docker_ExecContainer_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(DockerServer).ExecContainer(&grpc.GenericServerStream[ExecContainerRequest, ExecContainerResponse]{ServerStream: stream})
-}
-
-// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
-type Docker_ExecContainerServer = grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]
-
 // Docker_ServiceDesc is the grpc.ServiceDesc for Docker service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -754,17 +790,22 @@ var Docker_ServiceDesc = grpc.ServiceDesc{
 		},
 	},
 	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "PullImage",
-			Handler:       _Docker_PullImage_Handler,
-			ServerStreams: true,
-		},
 		{
 			StreamName:    "ExecContainer",
 			Handler:       _Docker_ExecContainer_Handler,
 			ServerStreams: true,
 			ClientStreams: true,
 		},
+		{
+			StreamName:    "ContainerLogs",
+			Handler:       _Docker_ContainerLogs_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "PullImage",
+			Handler:       _Docker_PullImage_Handler,
+			ServerStreams: true,
+		},
 	},
 	Metadata: "internal/machine/api/pb/docker.proto",
 }

internal/machine/docker/server.go

@@ -47,6 +47,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/emptypb"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 var fullDockerIDRegex = regexp.MustCompile(`^[a-f0-9]{64}$`)
@@ -1015,6 +1016,97 @@ func (s *Server) RemoveServiceContainer(ctx context.Context, req *pb.RemoveConta
 	return resp, nil
 }
 
+// logsHeartbeatInterval is the interval at which heartbeat entries are sent when there are no logs to stream.
+const logsHeartbeatInterval = 200 * time.Millisecond
+
+// ContainerLogs streams logs from a container.
+func (s *Server) ContainerLogs(
+	req *pb.ContainerLogsRequest, stream grpc.ServerStreamingServer[pb.ContainerLogEntry],
+) error {
+	// Stream context is cancelled when the client has disconnected or the stream has ended.
+	ctx := stream.Context()
+
+	opts := ContainerLogsOptions{
+		ContainerID: req.ContainerId,
+		Follow:      req.Follow,
+		Tail:        int(req.Tail),
+		Since:       req.Since,
+		Until:       req.Until,
+	}
+
+	logsCh, err := s.service.ContainerLogs(ctx, opts)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return status.Error(codes.NotFound, err.Error())
+		}
+		return status.Errorf(codes.Internal, "get container logs: %v", err)
+	}
+
+	log := slog.With("container_id", req.ContainerId, "stream_id", fmt.Sprintf("%p", stream)[2:])
+	log.Debug("Starting container logs streaming.",
+		"follow", req.Follow, "tail", req.Tail, "since", req.Since, "until", req.Until)
+
+	// Heartbeats are needed only when following logs to let the client know when there are no new log entries
+	// to allow it to advance the watermark of last received log timestamp.
+	var heartbeatCh <-chan time.Time
+	if req.Follow {
+		heartbeatTicker := time.NewTicker(logsHeartbeatInterval)
+		defer heartbeatTicker.Stop()
+		heartbeatCh = heartbeatTicker.C
+	}
+
+	started := time.Now()
+	lastSent := time.Time{}
+
+	for {
+		select {
+		case entry, ok := <-logsCh:
+			if !ok {
+				// Channel closed, no more log entries.
+				return nil
+			}
+
+			if entry.Err != nil {
+				return status.Error(codes.Internal, entry.Err.Error())
+			}
+
+			pbEntry := &pb.ContainerLogEntry{
+				Stream:    api.LogStreamTypeToProto(entry.Stream),
+				Timestamp: timestamppb.New(entry.Timestamp),
+				Message:   entry.Message,
+			}
+			if err = stream.Send(pbEntry); err != nil {
+				return status.Errorf(codes.Internal, "send log entry: %v", err)
+			}
+			lastSent = entry.Timestamp
+
+		case now := <-heartbeatCh:
+			// Only send heartbeat if no log entries have been sent since the last heartbeat interval or
+			// if no log entries have been sent at all for at least a heartbeat interval since starting.
+			if now.Sub(lastSent) < logsHeartbeatInterval ||
+				(lastSent.IsZero() && now.Sub(started) < logsHeartbeatInterval) {
+				continue
+			}
+
+			// Use the timestamp one heartbeat in the past to be conservative. This reduces the chance of sending
+			// a timestamp that is greater than a log entry currently being parsed but not yet sent, which would
+			// cause the client to incorrectly believe it has received all logs up to that point.
+			heartbeat := &pb.ContainerLogEntry{
+				Stream:    pb.ContainerLogEntry_HEARTBEAT,
+				Timestamp: timestamppb.New(now.Add(-logsHeartbeatInterval)),
+			}
+			if err = stream.Send(heartbeat); err != nil {
+				return status.Errorf(codes.Internal, "send log stream heartbeat: %v", err)
+			}
+			lastSent = heartbeat.Timestamp.AsTime()
+			log.Debug("Sent log stream heartbeat.", "timestamp", lastSent)
+
+		case <-ctx.Done():
+			return status.Error(codes.Canceled, ctx.Err().Error())
+		}
+	}
+}
+
 // receiveExecConfig receives and validates the initial exec configuration from the stream.
 func (s *Server) receiveExecConfig(stream pb.Docker_ExecContainerServer) (*pb.ExecConfig, api.ExecOptions, error) {
 	req, err := stream.Recv()

internal/machine/docker/service.go

@@ -1,18 +1,22 @@
 package docker
 
 import (
+	"bytes"
 	"context"
 	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"log/slog"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/jmoiron/sqlx"
 	"github.com/psviderski/uncloud/pkg/api"
 	"google.golang.org/grpc/codes"
@@ -150,3 +154,104 @@ func (s *Service) ListImages(ctx context.Context, opts image.ListOptions) (Image
 
 	return imagesResp, nil
 }
+
+// ContainerLogsOptions specifies parameters for ContainerLogs.
+type ContainerLogsOptions struct {
+	ContainerID string
+	Follow      bool
+	Tail        int
+	Since       string
+	Until       string
+}
+
+// ContainerLogs streams logs from a container and returns demultiplexed entries via a channel.
+// The channel is closed when streaming completes or context is cancelled.
+func (s *Service) ContainerLogs(ctx context.Context, opts ContainerLogsOptions) (<-chan api.ContainerLogEntry, error) {
+	dockerOpts := container.LogsOptions{
+		ShowStdout: true,
+		ShowStderr: true,
+		Follow:     opts.Follow,
+		Tail:       strconv.FormatInt(int64(opts.Tail), 10),
+		Since:      opts.Since,
+		Until:      opts.Until,
+		Timestamps: true,
+	}
+
+	reader, err := s.Client.ContainerLogs(ctx, opts.ContainerID, dockerOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	outCh := make(chan api.ContainerLogEntry)
+	stdoutWriter := &logsChannelWriter{ctx: ctx, ch: outCh, isStderr: false}
+	stderrWriter := &logsChannelWriter{ctx: ctx, ch: outCh, isStderr: true}
+
+	// Wrap the context in a cancellable one to unblock the second goroutine below when StdCopy completes.
+	ctx, cancel := context.WithCancel(ctx)
+
+	// Run StdCopy in a goroutine to be able to handle context cancellation.
+	go func() {
+		defer close(outCh)
+		defer cancel()
+
+		// StdCopy is blocking and will return when the reader is closed in another goroutine below or on error.
+		if _, err := stdcopy.StdCopy(stdoutWriter, stderrWriter, reader); err != nil {
+			// Send error as the last entry.
+			select {
+			case outCh <- api.ContainerLogEntry{Err: fmt.Errorf("demultiplex container logs: %w", err)}:
+			case <-ctx.Done():
+			}
+		}
+	}()
+
+	// Close the reader when the context is done to cancel StdCopy if it's still running.
+	go func() {
+		<-ctx.Done()
+		reader.Close()
+	}()
+
+	return outCh, nil
+}
+
+// logsChannelWriter is a writer for stdcopy.StdCopy that sends demultiplexed container logs to a channel.
+type logsChannelWriter struct {
+	ctx      context.Context
+	ch       chan<- api.ContainerLogEntry
+	isStderr bool
+}
+
+func (w *logsChannelWriter) Write(data []byte) (n int, err error) {
+	// Parse timestamp and message from the demultiplexed Docker log payload if the data looks like it contains one.
+	// Format: 2025-01-01T00:00:00.000000000Z message
+	timestamp := time.Time{}
+	message := data
+	if len(data) > 30 && data[4] == '-' && data[7] == '-' && data[10] == 'T' {
+		timestampPart, messagePart, found := bytes.Cut(data, []byte(" "))
+		if found {
+			timestamp, err = time.Parse(time.RFC3339Nano, string(timestampPart))
+			if err != nil {
+				timestamp = time.Time{}
+			}
+			message = messagePart
+		}
+	}
+
+	entry := api.ContainerLogEntry{
+		Timestamp: timestamp,
+		// Clone is required because message is a slice into data, which stdcopy.StdCopy may reuse
+		// after Write returns but before the entry is consumed from the channel.
+		Message: bytes.Clone(message),
+	}
+	if w.isStderr {
+		entry.Stream = api.LogStreamStderr
+	} else {
+		entry.Stream = api.LogStreamStdout
+	}
+
+	select {
+	case w.ch <- entry:
+		return len(data), nil
+	case <-w.ctx.Done():
+		return 0, w.ctx.Err()
+	}
+}