feat: service logs command to stream service logs (#196)

* chore: fix landing logo shadow * feat: implement service logs command with colored output and strict ordering * revert Makefile * fix after rebase * move logs command under services with root shortcut * simplify server options for streaming logs and CLI flags * refactor ContainerLogs grpc server * update --tail flag * minor docker.proto * refactor client ServiceLogs and ContainerLogs * move ProxyMachinesContext from api to client pkg * minor refactor proto Stream * add api/logs * implement LogMerger * fix LogMerger to correctly use semaphore * increate inflish entries to 100 per stream * send heartbeats * refactor ContainerLogs to synchronise Send of entries and heartbeats to the steram * minor logmerege * remove ContainerName form ServiceLogEntryMetadata * refactor ContainerLogs into docker.Service * detect stalled container logs streams * update logmerger tests * fix comment in test * refactor LogMerger with options * uc logs: format one or multiple services * make LogMerger emit heartbeats, emit entries <= watermark, rewrite tests * update uc logs with new LogMerger * go mod tidy * fix after merge --------- Co-authored-by: Evgenii Orlov <evgenii.orlov@semrush.com>
2026-08-26 19:13:34 +00:00 · 2025-12-02 19:50:48 +10:00
parent 234985b57d
commit 79dc05cb66
25 changed files with 3657 additions and 1925 deletions
@@ -5,6 +5,7 @@ package api;
 option go_package = "github.com/psviderski/uncloud/internal/machine/api/pb";

 import "google/protobuf/empty.proto";
+import "google/protobuf/timestamp.proto";
 import "internal/machine/api/pb/common.proto";

 service Docker {
@@ -15,6 +16,9 @@ service Docker {
  rpc ListContainers(ListContainersRequest) returns (ListContainersResponse);
  rpc RemoveContainer(RemoveContainerRequest) returns (google.protobuf.Empty);

+  rpc ExecContainer(stream ExecContainerRequest) returns (stream ExecContainerResponse);
+  rpc ContainerLogs(ContainerLogsRequest) returns (stream ContainerLogEntry);
+
  rpc PullImage(PullImageRequest) returns (stream JSONMessage);
  rpc InspectImage(InspectImageRequest) returns (InspectImageResponse);
  // InspectRemoteImage returns the image metadata for an image in a remote registry using the machine's
@@ -30,8 +34,6 @@ service Docker {
  rpc InspectServiceContainer(InspectContainerRequest) returns (ServiceContainer);
  rpc ListServiceContainers(ListServiceContainersRequest) returns (ListServiceContainersResponse);
  rpc RemoveServiceContainer(RemoveContainerRequest) returns (google.protobuf.Empty);
-
-  rpc ExecContainer(stream ExecContainerRequest) returns (stream ExecContainerResponse);
 }

 message CreateContainerRequest {
@@ -94,6 +96,64 @@ message RemoveContainerRequest {
  bytes options = 2;
 }

+message ExecContainerRequest {
+  oneof payload {
+    // Initial configuration for the exec session. Must be sent as the first message.
+    ExecConfig config = 1;
+    // Raw stdin data to be written to the exec process.
+    bytes stdin = 2;
+    // TTY resize event (only used when TTY is enabled).
+    ResizeEvent resize = 3;
+  }
+}
+
+message ExecConfig {
+  // Container ID to execute the command in.
+  string container_id = 1;
+  // JSON serialised ExecOptions
+  bytes options = 2;
+}
+
+message ResizeEvent {
+  uint32 height = 1;
+  uint32 width = 2;
+}
+
+message ExecContainerResponse {
+  oneof payload {
+    // Exec instance ID returned after creating the exec.
+    string exec_id = 1;
+    // Raw stdout data from the exec process.
+    bytes stdout = 2;
+    // Raw stderr data from the exec process (only when TTY is disabled).
+    bytes stderr = 3;
+    // Exit code of the exec process. Sent as the final message.
+    int32 exit_code = 4;
+  }
+}
+
+message ContainerLogsRequest {
+  string container_id = 1;
+  // Options for logs retrieval.
+  bool follow = 2;
+  int32 tail = 3; // -1 means all
+  string since = 4; // https://www.rfc-editor.org/rfc/rfc3339.html timestamp or Go duration string
+  string until = 5; // https://www.rfc-editor.org/rfc/rfc3339.html timestamp or Go duration string
+}
+
+message ContainerLogEntry {
+  enum StreamType {
+    UNKNOWN = 0;
+    STDOUT = 1;
+    STDERR = 2;
+    HEARTBEAT = 3;
+  }
+  StreamType stream = 1;
+  google.protobuf.Timestamp timestamp = 2;
+  // Log line content. Empty for heartbeat entries.
+  bytes message = 3;
+}
+
 message PullImageRequest {
  string image = 1;
  // JSON serialised image.PullOptions.
@@ -216,39 +276,3 @@ message MachineServiceContainers {
  Metadata metadata = 1;
  repeated ServiceContainer containers = 2;
 }
-
-message ExecContainerRequest {
-  oneof payload {
-    // Initial configuration for the exec session. Must be sent as the first message.
-    ExecConfig config = 1;
-    // Raw stdin data to be written to the exec process.
-    bytes stdin = 2;
-    // TTY resize event (only used when TTY is enabled).
-    ResizeEvent resize = 3;
-  }
-}
-
-message ExecConfig {
-  // Container ID to execute the command in.
-  string container_id = 1;
-  // JSON serialised ExecOptions
-  bytes options = 2;
-}
-
-message ResizeEvent {
-  uint32 height = 1;
-  uint32 width = 2;
-}
-
-message ExecContainerResponse {
-  oneof payload {
-    // Exec instance ID returned after creating the exec.
-    string exec_id = 1;
-    // Raw stdout data from the exec process.
-    bytes stdout = 2;
-    // Raw stderr data from the exec process (only when TTY is disabled).
-    bytes stderr = 3;
-    // Exit code of the exec process. Sent as the final message.
-    int32 exit_code = 4;
-  }
-}
@@ -26,6 +26,8 @@ const (
 	Docker_StopContainer_FullMethodName           = "/api.Docker/StopContainer"
 	Docker_ListContainers_FullMethodName          = "/api.Docker/ListContainers"
 	Docker_RemoveContainer_FullMethodName         = "/api.Docker/RemoveContainer"
+	Docker_ExecContainer_FullMethodName           = "/api.Docker/ExecContainer"
+	Docker_ContainerLogs_FullMethodName           = "/api.Docker/ContainerLogs"
 	Docker_PullImage_FullMethodName               = "/api.Docker/PullImage"
 	Docker_InspectImage_FullMethodName            = "/api.Docker/InspectImage"
 	Docker_InspectRemoteImage_FullMethodName      = "/api.Docker/InspectRemoteImage"
@@ -37,7 +39,6 @@ const (
 	Docker_InspectServiceContainer_FullMethodName = "/api.Docker/InspectServiceContainer"
 	Docker_ListServiceContainers_FullMethodName   = "/api.Docker/ListServiceContainers"
 	Docker_RemoveServiceContainer_FullMethodName  = "/api.Docker/RemoveServiceContainer"
-	Docker_ExecContainer_FullMethodName           = "/api.Docker/ExecContainer"
 )

 // DockerClient is the client API for Docker service.
@@ -50,6 +51,8 @@ type DockerClient interface {
 	StopContainer(ctx context.Context, in *StopContainerRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
 	ListContainers(ctx context.Context, in *ListContainersRequest, opts ...grpc.CallOption) (*ListContainersResponse, error)
 	RemoveContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
+	ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error)
+	ContainerLogs(ctx context.Context, in *ContainerLogsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ContainerLogEntry], error)
 	PullImage(ctx context.Context, in *PullImageRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[JSONMessage], error)
 	InspectImage(ctx context.Context, in *InspectImageRequest, opts ...grpc.CallOption) (*InspectImageResponse, error)
 	// InspectRemoteImage returns the image metadata for an image in a remote registry using the machine's
@@ -63,7 +66,6 @@ type DockerClient interface {
 	InspectServiceContainer(ctx context.Context, in *InspectContainerRequest, opts ...grpc.CallOption) (*ServiceContainer, error)
 	ListServiceContainers(ctx context.Context, in *ListServiceContainersRequest, opts ...grpc.CallOption) (*ListServiceContainersResponse, error)
 	RemoveServiceContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
-	ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error)
 }

 type dockerClient struct {
@@ -134,9 +136,41 @@ func (c *dockerClient) RemoveContainer(ctx context.Context, in *RemoveContainerR
 	return out, nil
 }

+func (c *dockerClient) ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[0], Docker_ExecContainer_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[ExecContainerRequest, ExecContainerResponse]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ExecContainerClient = grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse]
+
+func (c *dockerClient) ContainerLogs(ctx context.Context, in *ContainerLogsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ContainerLogEntry], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[1], Docker_ContainerLogs_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[ContainerLogsRequest, ContainerLogEntry]{ClientStream: stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ContainerLogsClient = grpc.ServerStreamingClient[ContainerLogEntry]
+
 func (c *dockerClient) PullImage(ctx context.Context, in *PullImageRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[JSONMessage], error) {
 	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
-	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[0], Docker_PullImage_FullMethodName, cOpts...)
+	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[2], Docker_PullImage_FullMethodName, cOpts...)
 	if err != nil {
 		return nil, err
 	}
@@ -253,19 +287,6 @@ func (c *dockerClient) RemoveServiceContainer(ctx context.Context, in *RemoveCon
 	return out, nil
 }

-func (c *dockerClient) ExecContainer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse], error) {
-	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
-	stream, err := c.cc.NewStream(ctx, &Docker_ServiceDesc.Streams[1], Docker_ExecContainer_FullMethodName, cOpts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &grpc.GenericClientStream[ExecContainerRequest, ExecContainerResponse]{ClientStream: stream}
-	return x, nil
-}
-
-// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
-type Docker_ExecContainerClient = grpc.BidiStreamingClient[ExecContainerRequest, ExecContainerResponse]
-
 // DockerServer is the server API for Docker service.
 // All implementations must embed UnimplementedDockerServer
 // for forward compatibility.
@@ -276,6 +297,8 @@ type DockerServer interface {
 	StopContainer(context.Context, *StopContainerRequest) (*emptypb.Empty, error)
 	ListContainers(context.Context, *ListContainersRequest) (*ListContainersResponse, error)
 	RemoveContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error)
+	ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error
+	ContainerLogs(*ContainerLogsRequest, grpc.ServerStreamingServer[ContainerLogEntry]) error
 	PullImage(*PullImageRequest, grpc.ServerStreamingServer[JSONMessage]) error
 	InspectImage(context.Context, *InspectImageRequest) (*InspectImageResponse, error)
 	// InspectRemoteImage returns the image metadata for an image in a remote registry using the machine's
@@ -289,7 +312,6 @@ type DockerServer interface {
 	InspectServiceContainer(context.Context, *InspectContainerRequest) (*ServiceContainer, error)
 	ListServiceContainers(context.Context, *ListServiceContainersRequest) (*ListServiceContainersResponse, error)
 	RemoveServiceContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error)
-	ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error
 	mustEmbedUnimplementedDockerServer()
 }

@@ -318,6 +340,12 @@ func (UnimplementedDockerServer) ListContainers(context.Context, *ListContainers
 func (UnimplementedDockerServer) RemoveContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method RemoveContainer not implemented")
 }
+func (UnimplementedDockerServer) ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method ExecContainer not implemented")
+}
+func (UnimplementedDockerServer) ContainerLogs(*ContainerLogsRequest, grpc.ServerStreamingServer[ContainerLogEntry]) error {
+	return status.Errorf(codes.Unimplemented, "method ContainerLogs not implemented")
+}
 func (UnimplementedDockerServer) PullImage(*PullImageRequest, grpc.ServerStreamingServer[JSONMessage]) error {
 	return status.Errorf(codes.Unimplemented, "method PullImage not implemented")
 }
@@ -351,9 +379,6 @@ func (UnimplementedDockerServer) ListServiceContainers(context.Context, *ListSer
 func (UnimplementedDockerServer) RemoveServiceContainer(context.Context, *RemoveContainerRequest) (*emptypb.Empty, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method RemoveServiceContainer not implemented")
 }
-func (UnimplementedDockerServer) ExecContainer(grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]) error {
-	return status.Errorf(codes.Unimplemented, "method ExecContainer not implemented")
-}
 func (UnimplementedDockerServer) mustEmbedUnimplementedDockerServer() {}
 func (UnimplementedDockerServer) testEmbeddedByValue()                {}

@@ -483,6 +508,24 @@ func _Docker_RemoveContainer_Handler(srv interface{}, ctx context.Context, dec f
 	return interceptor(ctx, in, info, handler)
 }

+func _Docker_ExecContainer_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(DockerServer).ExecContainer(&grpc.GenericServerStream[ExecContainerRequest, ExecContainerResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ExecContainerServer = grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]
+
+func _Docker_ContainerLogs_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(ContainerLogsRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(DockerServer).ContainerLogs(m, &grpc.GenericServerStream[ContainerLogsRequest, ContainerLogEntry]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type Docker_ContainerLogsServer = grpc.ServerStreamingServer[ContainerLogEntry]
+
 func _Docker_PullImage_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(PullImageRequest)
 	if err := stream.RecvMsg(m); err != nil {
@@ -674,13 +717,6 @@ func _Docker_RemoveServiceContainer_Handler(srv interface{}, ctx context.Context
 	return interceptor(ctx, in, info, handler)
 }

-func _Docker_ExecContainer_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(DockerServer).ExecContainer(&grpc.GenericServerStream[ExecContainerRequest, ExecContainerResponse]{ServerStream: stream})
-}
-
-// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
-type Docker_ExecContainerServer = grpc.BidiStreamingServer[ExecContainerRequest, ExecContainerResponse]
-
 // Docker_ServiceDesc is the grpc.ServiceDesc for Docker service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -754,17 +790,22 @@ var Docker_ServiceDesc = grpc.ServiceDesc{
 		},
 	},
 	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "PullImage",
-			Handler:       _Docker_PullImage_Handler,
-			ServerStreams: true,
-		},
 		{
 			StreamName:    "ExecContainer",
 			Handler:       _Docker_ExecContainer_Handler,
 			ServerStreams: true,
 			ClientStreams: true,
 		},
+		{
+			StreamName:    "ContainerLogs",
+			Handler:       _Docker_ContainerLogs_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "PullImage",
+			Handler:       _Docker_PullImage_Handler,
+			ServerStreams: true,
+		},
 	},
 	Metadata: "internal/machine/api/pb/docker.proto",
 }