feat: service logs command to stream service logs (#196)

* chore: fix landing logo shadow

* feat: implement service logs command with colored output and strict ordering

* revert Makefile

* fix after rebase

* move logs command under services with root shortcut

* simplify server options for streaming logs and CLI flags

* refactor ContainerLogs grpc server

* update --tail flag

* minor docker.proto

* refactor client ServiceLogs and ContainerLogs

* move ProxyMachinesContext from api to client pkg

* minor refactor proto Stream

* add api/logs

* implement LogMerger

* fix LogMerger to correctly use semaphore

* increate inflish entries to 100 per stream

* send heartbeats

* refactor ContainerLogs to synchronise Send of entries and heartbeats to the steram

* minor logmerege

* remove ContainerName form ServiceLogEntryMetadata

* refactor ContainerLogs into docker.Service

* detect stalled container logs streams

* update logmerger tests

* fix comment in test

* refactor LogMerger with options

* uc logs: format one or multiple services

* make LogMerger emit heartbeats, emit entries <= watermark, rewrite tests

* update uc logs with new LogMerger

* go mod tidy

* fix after merge

---------

Co-authored-by: Evgenii Orlov <evgenii.orlov@semrush.com>

cmd/uncloud/caddy/config.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/alecthomas/chroma/v2/quick"
 	"github.com/psviderski/uncloud/internal/cli"
-	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/spf13/cobra"
 )
 
@@ -46,7 +45,7 @@ func runConfig(ctx context.Context, uncli *cli.CLI, opts configOptions) error {
 
 	if opts.machine != "" {
 		// If a specific machine is requested, use it to get the Caddy configuration.
-		ctx, _, err = api.ProxyMachinesContext(ctx, clusterClient, []string{opts.machine})
+		ctx, _, err = clusterClient.ProxyMachinesContext(ctx, []string{opts.machine})
 		if err != nil {
 			return err
 		}

cmd/uncloud/machine/rm.go

@@ -55,7 +55,7 @@ func remove(ctx context.Context, uncli *cli.CLI, nameOrID string, opts removeOpt
 	defer client.Close()
 
 	// Verify the machine exists and list all service containers on it including stopped ones.
-	mctx, machines, err := api.ProxyMachinesContext(ctx, client, []string{nameOrID})
+	mctx, machines, err := client.ProxyMachinesContext(ctx, []string{nameOrID})
 	if err != nil {
 		return err
 	}

cmd/uncloud/machine/root.go

@@ -8,7 +8,7 @@ func NewRootCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "machine",
 		Aliases: []string{"m"},
-		Short:   "Manage machines in an Uncloud cluster.",
+		Short:   "Manage machines in the cluster.",
 	}
 	cmd.AddCommand(
 		NewAddCommand(),

cmd/uncloud/main.go

@@ -108,9 +108,9 @@ func main() {
 	})
 
 	cmd.AddCommand(
+		NewBuildCommand(),
 		NewDeployCommand(),
 		NewDocsCommand(),
-		NewBuildCommand(),
 		NewImagesCommand(),
 		NewPsCommand(),
 		caddy.NewRootCommand(),
@@ -122,6 +122,7 @@ func main() {
 		service.NewExecCommand(),
 		service.NewInspectCommand(),
 		service.NewListCommand(),
+		service.NewLogsCommand(),
 		service.NewRmCommand(),
 		service.NewRunCommand(),
 		service.NewScaleCommand(),

cmd/uncloud/ps.go

@@ -12,7 +12,6 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/psviderski/uncloud/internal/cli"
-	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client"
 )
 
@@ -46,12 +45,14 @@ This command provides a comprehensive overview of all running containers that ar
 making it easy to see the distribution and status of containers across the cluster.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if opts.sortBy != sortByService && opts.sortBy != sortByMachine && opts.sortBy != sortByHealth {
-				return fmt.Errorf("invalid value for --sort: %q, must be one of '%s', '%s' or '%s'", opts.sortBy, sortByService, sortByMachine, sortByHealth)
+				return fmt.Errorf("invalid value for --sort: %q, must be one of '%s', '%s' or '%s'", opts.sortBy,
+					sortByService, sortByMachine, sortByHealth)
 			}
 			return runPs(cmd, opts)
 		},
 	}
-	cmd.Flags().StringVarP(&opts.sortBy, "sort", "s", sortByService, "Sort containers by 'service', 'machine' or 'health'")
+	cmd.Flags().StringVarP(&opts.sortBy, "sort", "s", sortByService,
+		"Sort containers by 'service', 'machine' or 'health'")
 	return cmd
 }
 
@@ -176,7 +177,7 @@ func printContainers(containers []containerInfo) error {
 }
 
 func collectContainers(ctx context.Context, cli *client.Client) ([]containerInfo, error) {
-	listCtx, machines, err := api.ProxyMachinesContext(ctx, cli, nil)
+	listCtx, machines, err := cli.ProxyMachinesContext(ctx, nil)
 	if err != nil {
 		return nil, fmt.Errorf("proxy machines context: %w", err)
 	}
@@ -206,7 +207,8 @@ func collectContainers(ctx context.Context, cli *client.Client) ([]containerInfo
 			return nil, fmt.Errorf("something went wrong with gRPC proxy: metadata is missing for a machine response")
 		}
 		if msc.Metadata != nil && msc.Metadata.Error != "" {
-			client.PrintWarning(fmt.Sprintf("failed to list containers on machine %s: %s", machineName, msc.Metadata.Error))
+			client.PrintWarning(fmt.Sprintf("failed to list containers on machine %s: %s", machineName,
+				msc.Metadata.Error))
 			continue
 		}
 

cmd/uncloud/service/logs.go

@@ -0,0 +1,279 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/charmbracelet/lipgloss"
+	mapset "github.com/deckarep/golang-set/v2"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/psviderski/uncloud/internal/cli"
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/psviderski/uncloud/pkg/client"
+	"github.com/spf13/cobra"
+)
+
+type logsOptions struct {
+	follow bool
+	tail   string
+	since  string
+	until  string
+}
+
+func NewLogsCommand() *cobra.Command {
+	var options logsOptions
+
+	cmd := &cobra.Command{
+		Use:     "logs SERVICE [SERVICE...]",
+		Aliases: []string{"log"},
+		Short:   "View service logs.",
+		Long:    "View logs from all replicas of the specified service(s) across all machines in the cluster.",
+		Args:    cobra.MinimumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			uncli := cmd.Context().Value("cli").(*cli.CLI)
+			return streamLogs(cmd.Context(), uncli, args, options)
+		},
+	}
+
+	cmd.Flags().BoolVarP(&options.follow, "follow", "f", false,
+		"Continually stream new logs.")
+	cmd.Flags().StringVarP(&options.tail, "tail", "n", "100",
+		"Show the most recent logs and limit the number of lines shown per replica. Use 'all' to show all logs.")
+	cmd.Flags().StringVar(&options.since, "since", "",
+		"Show logs generated on or after the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.\n"+
+			"Examples:\n"+
+			"  --since 2m30s                      Relative duration (2 minutes 30 seconds ago)\n"+
+			"  --since 1h                         Relative duration (1 hour ago)\n"+
+			"  --since 2025-11-24                 RFC 3339 date only (midnight using local timezone)\n"+
+			"  --since 2024-05-14T22:50:00        RFC 3339 date/time using local timezone\n"+
+			"  --since 2024-01-31T10:30:00Z       RFC 3339 date/time in UTC\n"+
+			"  --since 1763953966                 Unix timestamp (seconds since January 1, 1970)")
+	cmd.Flags().StringVar(&options.until, "until", "",
+		"Show logs generated before the given timestamp. Accepts relative duration, RFC 3339 date, or Unix timestamp.\n"+
+			"See --since for examples.")
+
+	return cmd
+}
+
+func streamLogs(ctx context.Context, uncli *cli.CLI, serviceNames []string, opts logsOptions) error {
+	// Parse tail option.
+	tail := -1
+	if opts.tail != "all" {
+		tailInt, err := strconv.Atoi(opts.tail)
+		if err != nil {
+			return fmt.Errorf("invalid --tail value '%s': %w", opts.tail, err)
+		}
+		tail = tailInt
+	}
+
+	c, err := uncli.ConnectCluster(ctx)
+	if err != nil {
+		return fmt.Errorf("connect to cluster: %w", err)
+	}
+	defer c.Close()
+
+	logsOpts := api.ServiceLogsOptions{
+		Follow: opts.follow,
+		Tail:   tail,
+		Since:  opts.since,
+		Until:  opts.until,
+	}
+
+	// Collect log streams from all services.
+	machineIDsSet := mapset.NewSet[string]()
+	svcStreams := make([]<-chan api.ServiceLogEntry, 0, len(serviceNames))
+	for _, serviceName := range serviceNames {
+		// TODO: set Heartbeats in the opts.
+		svc, ch, err := c.ServiceLogs(ctx, serviceName, logsOpts)
+		if err != nil {
+			return fmt.Errorf("stream logs for service '%s': %w", serviceName, err)
+		}
+		svcStreams = append(svcStreams, ch)
+
+		machineIDs := svc.MachineIDs()
+		machineIDsSet.Append(machineIDs...)
+	}
+
+	var stream <-chan api.ServiceLogEntry
+	if len(serviceNames) == 1 {
+		stream = svcStreams[0]
+	} else {
+		// Merge all service streams into a single sorted stream without stall detection as its handled per-service.
+		merger := client.NewLogMerger(svcStreams, client.LogMergerOptions{})
+		stream = merger.Stream()
+	}
+
+	// Fetch machine names for all machines (machineIDsSet) service containers are running on.
+	machines, err := c.ListMachines(ctx, &api.MachineFilter{NamesOrIDs: machineIDsSet.ToSlice()})
+	if err != nil {
+		return fmt.Errorf("list machines: %w", err)
+	}
+	machineNames := make([]string, 0, len(machines))
+	for _, m := range machines {
+		machineNames = append(machineNames, m.Machine.Name)
+	}
+
+	formatter := newLogFormatter(machineNames, serviceNames)
+
+	// Print merged logs.
+	for entry := range stream {
+		if entry.Err != nil {
+			formatter.printError(entry)
+			continue
+		}
+		formatter.printEntry(entry)
+	}
+
+	return nil
+}
+
+// Available colors for machine/service differentiation.
+var colorPalette = []lipgloss.Color{
+	lipgloss.Color("10"), // Bright green
+	lipgloss.Color("11"), // Bright yellow
+	lipgloss.Color("12"), // Bright blue
+	lipgloss.Color("13"), // Bright magenta
+	lipgloss.Color("14"), // Bright cyan
+	lipgloss.Color("2"),  // Green
+	lipgloss.Color("3"),  // Yellow
+	lipgloss.Color("4"),  // Blue
+	lipgloss.Color("5"),  // Magenta
+	lipgloss.Color("6"),  // Cyan
+}
+
+// logFormatter handles formatting and printing of log entries with dynamic column alignment.
+type logFormatter struct {
+	machineNames []string
+	serviceNames []string
+
+	maxMachineWidth int
+	maxServiceWidth int
+}
+
+func newLogFormatter(machineNames, serviceNames []string) *logFormatter {
+	slices.Sort(machineNames)
+	slices.Sort(serviceNames)
+
+	maxMachineWidth := 0
+	for _, name := range machineNames {
+		if len(name) > maxMachineWidth {
+			maxMachineWidth = len(name)
+		}
+	}
+
+	maxServiceWidth := 0
+	for _, name := range serviceNames {
+		if len(name) > maxServiceWidth {
+			maxServiceWidth = len(name)
+		}
+	}
+
+	return &logFormatter{
+		machineNames:    machineNames,
+		serviceNames:    serviceNames,
+		maxMachineWidth: maxMachineWidth,
+		maxServiceWidth: maxServiceWidth,
+	}
+}
+
+// formatTimestamp formats timestamp using local timezone.
+func (f *logFormatter) formatTimestamp(t time.Time) string {
+	dimStyle := lipgloss.NewStyle().Faint(true)
+	t = t.In(time.Local)
+	return dimStyle.Render(t.Format(time.StampMilli))
+}
+
+func (f *logFormatter) formatMachine(name string) string {
+	style := lipgloss.NewStyle().Bold(true).PaddingRight(f.maxMachineWidth - len(name))
+
+	if len(f.serviceNames) == 1 {
+		// Machine name is coloured for single-service logs.
+		i := slices.Index(f.machineNames, name)
+		if i == -1 {
+			f.machineNames = append(f.machineNames, name)
+			i = len(f.machineNames) - 1
+		}
+
+		style = style.Foreground(colorPalette[i%len(colorPalette)])
+	}
+
+	return style.Render(name)
+}
+
+func (f *logFormatter) formatServiceContainer(serviceName, containerID string) string {
+	styleService := lipgloss.NewStyle().Bold(true).PaddingRight(f.maxServiceWidth - len(serviceName))
+	styleContainer := lipgloss.NewStyle().Faint(true)
+
+	if len(f.serviceNames) > 1 {
+		// Service name is coloured for multi-service logs.
+		i := slices.Index(f.serviceNames, serviceName)
+		if i == -1 {
+			f.serviceNames = append(f.serviceNames, serviceName)
+			i = len(f.serviceNames) - 1
+		}
+
+		styleService = styleService.Foreground(colorPalette[i%len(colorPalette)])
+	}
+
+	return styleService.Render(serviceName) + styleContainer.Render("["+containerID[:5]+"]")
+}
+
+// printEntry prints a single log entry with proper formatting.
+func (f *logFormatter) printEntry(entry api.ServiceLogEntry) {
+	if entry.Stream != api.LogStreamStdout && entry.Stream != api.LogStreamStderr {
+		return
+	}
+
+	var output strings.Builder
+
+	// Timestamp
+	output.WriteString(f.formatTimestamp(entry.Timestamp))
+	output.WriteString(" ")
+
+	// Machine name
+	output.WriteString(f.formatMachine(entry.Metadata.MachineName))
+	output.WriteString(" ")
+
+	// Service[container_id]
+	output.WriteString(f.formatServiceContainer(entry.Metadata.ServiceName, entry.Metadata.ContainerID))
+	output.WriteString(" ")
+
+	// Message
+	output.Write(entry.Message)
+
+	// Print to appropriate stream.
+	if entry.Stream == api.LogStreamStderr {
+		fmt.Fprint(os.Stderr, output.String())
+	} else {
+		fmt.Print(output.String())
+	}
+}
+
+// printError prints an error entry (e.g., stalled stream warning).
+func (f *logFormatter) printError(entry api.ServiceLogEntry) {
+	if entry.Metadata.ContainerID != "" {
+		msg := fmt.Sprintf("WARNING: log stream from %s[%s] on machine '%s'",
+			entry.Metadata.ServiceName,
+			stringid.TruncateID(entry.Metadata.ContainerID),
+			entry.Metadata.MachineName)
+
+		if errors.Is(entry.Err, api.ErrLogStreamStalled) {
+			msg += " stopped responding"
+		} else {
+			msg += fmt.Sprintf(": %v", entry.Err)
+		}
+
+		style := lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("11")) // Bold bright yellow
+		fmt.Fprintln(os.Stderr, style.Render(msg))
+	} else {
+		msg := fmt.Sprintf("ERROR: %v", entry.Err)
+		style := lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("9")) // Bold bright red
+		fmt.Fprintln(os.Stderr, style.Render(msg))
+	}
+}

cmd/uncloud/service/root.go

@@ -8,15 +8,16 @@ func NewRootCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "service",
 		Aliases: []string{"svc"},
-		Short:   "Manage services in an Uncloud cluster.",
+		Short:   "Manage services in the cluster.",
 	}
 	cmd.AddCommand(
+		NewExecCommand(),
 		NewInspectCommand(),
 		NewListCommand(),
+		NewLogsCommand(),
 		NewRmCommand(),
 		NewRunCommand(),
 		NewScaleCommand(),
-		NewExecCommand(),
 	)
 	return cmd
 }

cmd/uncloud/volume/root.go

@@ -7,7 +7,7 @@ import (
 func NewRootCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "volume",
-		Short: "Manage volumes in an Uncloud cluster.",
+		Short: "Manage volumes in the cluster.",
 	}
 	cmd.AddCommand(
 		NewCreateCommand(),