From 303c8e4506d5f57dcb75126e58c52f69d986f785 Mon Sep 17 00:00:00 2001 From: Pasha Sviderski Date: Wed, 22 Apr 2026 13:45:09 +1000 Subject: [PATCH] feat(deploy): print last logs from new container when fails to become healthy --- cmd/uncloud/deploy.go | 57 +++++++++++++++++------- pkg/client/deploy/operation/container.go | 39 ++++++++++++---- pkg/client/deploy/operation/predeploy.go | 4 ++ 3 files changed, 76 insertions(+), 24 deletions(-) diff --git a/cmd/uncloud/deploy.go b/cmd/uncloud/deploy.go index f9961c24..dabf839b 100644 --- a/cmd/uncloud/deploy.go +++ b/cmd/uncloud/deploy.go @@ -9,6 +9,7 @@ import ( "charm.land/lipgloss/v2" composecli "github.com/compose-spec/compose-go/v2/cli" "github.com/docker/compose/v2/pkg/progress" + "github.com/docker/docker/pkg/stringid" "github.com/psviderski/uncloud/internal/cli" "github.com/psviderski/uncloud/internal/cli/logs" "github.com/psviderski/uncloud/internal/cli/tui" @@ -20,10 +21,6 @@ import ( "github.com/spf13/cobra" ) -// failedContainerLogsTail is the number of recent log lines to print from a failed container to give the user immediate -// context without requiring a follow-up 'uc logs' invocation. -const failedContainerLogsTail = 10 - type deployOptions struct { cli.BuildServicesOptions @@ -238,34 +235,46 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error { }, uncli.ProgressOut(), title) if err != nil { fmt.Println() + + tail := failedContainerLogsTail() if hookErr, ok := errors.AsType[*operation.PreDeployHookError](err); ok { - printPreDeployHookLogs(ctx, clusterClient, hookErr) + printFailedContainerLogs(ctx, clusterClient, + hookErr.ServiceName, hookErr.ContainerID, hookErr.MachineName, tail, + fmt.Sprintf("Last %d log lines from failed pre-deploy hook:", tail)) + fmt.Println() + } else if startErr, ok := errors.AsType[*operation.ContainerHealthError](err); ok { + printFailedContainerLogs(ctx, clusterClient, + startErr.ServiceName, startErr.ContainerID, startErr.MachineName, tail, + fmt.Sprintf("Last %d log lines from failed container:", tail)) fmt.Println() } + return err } return nil } -// printPreDeployHookLogs fetches the last log lines from the failed pre-deploy hook container and prints them using -// the standard log formatter. -func printPreDeployHookLogs(ctx context.Context, cli *client.Client, hookErr *operation.PreDeployHookError) { - _, ch, err := cli.ServiceLogs(ctx, hookErr.ServiceName, api.ServiceLogsOptions{ - Containers: []string{hookErr.ContainerID}, - Machines: []string{hookErr.MachineName}, - Tail: failedContainerLogsTail, +// printFailedContainerLogs fetches the last tail log lines from a container that failed during deployment and prints +// them using the standard log formatter under the provided header. +func printFailedContainerLogs( + ctx context.Context, cli *client.Client, serviceName, containerID, machineName string, tail int, header string, +) { + _, ch, err := cli.ServiceLogs(ctx, serviceName, api.ServiceLogsOptions{ + Containers: []string{containerID}, + Machines: []string{machineName}, + Tail: tail, }) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to fetch pre-deploy hook logs: %v\n", err) - fmt.Fprintf(os.Stderr, "You can try manually with: uc logs %s\n", hookErr.ServiceName) + shortCtrID := stringid.TruncateID(containerID) + fmt.Fprintf(os.Stderr, "Failed to fetch container logs '%s/%s': %v\n", serviceName, shortCtrID, err) + fmt.Fprintf(os.Stderr, "You can try manually with: uc logs %s/%s\n", serviceName, shortCtrID) return } - header := fmt.Sprintf("Last %d log lines from failed pre-deploy hook:", failedContainerLogsTail) fmt.Println(tui.BoldRed.Render(header)) logsEmpty := true - formatter := logs.NewFormatter([]string{hookErr.MachineName}, []string{hookErr.ServiceName}, false) + formatter := logs.NewFormatter([]string{machineName}, []string{serviceName}, false) for entry := range ch { logsEmpty = false formatter.PrintEntry(entry) @@ -275,3 +284,19 @@ func printPreDeployHookLogs(ctx context.Context, cli *client.Client, hookErr *op fmt.Println("") } } + +// defaultFailedContainerLogsTail is the default number of recent log lines to print from a failed container to give +// the user immediate context without requiring a follow-up 'uc logs' invocation. +// Overridable via UNCLOUD_FAILED_CONTAINER_LOGS_TAIL. +const defaultFailedContainerLogsTail = 10 + +// failedContainerLogsTail returns the number of log lines to fetch from a failed container, honouring the +// UNCLOUD_FAILED_CONTAINER_LOGS_TAIL environment variable override when set and valid. +func failedContainerLogsTail() int { + if v := os.Getenv("UNCLOUD_FAILED_CONTAINER_LOGS_TAIL"); v != "" { + if tail, err := logs.Tail(v); err == nil && (tail == -1 || tail > 0) { + return tail + } + } + return defaultFailedContainerLogsTail +} diff --git a/pkg/client/deploy/operation/container.go b/pkg/client/deploy/operation/container.go index 8b81efe5..de651eff 100644 --- a/pkg/client/deploy/operation/container.go +++ b/pkg/client/deploy/operation/container.go @@ -13,6 +13,18 @@ import ( "github.com/psviderski/uncloud/pkg/api" ) +// ContainerHealthError indicates that a service container failed to become healthy during deployment. +type ContainerHealthError struct { + error + ServiceName string + ContainerID string + MachineName string +} + +func (e *ContainerHealthError) Unwrap() error { + return e.error +} + // RunContainerOperation creates and starts a new container on a specific machine. type RunContainerOperation struct { ServiceID string @@ -43,8 +55,13 @@ func (o *RunContainerOperation) Execute(ctx context.Context, cli Client) error { opts := api.WaitContainerHealthyOptions{MonitorPeriod: o.Spec.UpdateConfig.MonitorPeriod} if err = cli.WaitContainerHealthy(ctx, o.ServiceID, resp.ID, opts); err != nil { - return fmt.Errorf("container '%s/%s' failed to become healthy: %w", - o.Spec.Name, stringid.TruncateID(resp.ID), err) + return &ContainerHealthError{ + error: fmt.Errorf("container '%s/%s' failed to become healthy: %w", + o.Spec.Name, stringid.TruncateID(resp.ID), err), + ServiceName: o.Spec.Name, + ContainerID: resp.ID, + MachineName: o.MachineName, + } } return nil @@ -187,7 +204,6 @@ func (o *ReplaceContainerOperation) Execute(ctx context.Context, cli Client) err if err = cli.WaitContainerHealthy(newCtx, o.ServiceID, resp.ID, opts); err != nil { // New container failed to become healthy. Stop it and roll back to the previous container. // Don't remove the new stopped container to allow users to inspect logs and state. - // TODO: collect logs from the new container and include in the error message to speed up debugging. // Use context without progress to not overwrite the container Unhealthy status with Stopped. ctxWithoutProgress := progress.WithContextWriter(ctx, nil) @@ -196,21 +212,28 @@ func (o *ReplaceContainerOperation) Execute(ctx context.Context, cli Client) err newCtr := fmt.Sprintf("%s/%s", o.Spec.Name, stringid.TruncateID(resp.ID)) healthErr := fmt.Errorf( "new container '%s' failed to become healthy: %w. "+ - "It's stopped and available for inspection. Fetch logs with 'uc logs %s'", - newCtr, err, o.Spec.Name, + "It's stopped and available for inspection. View logs with 'uc logs %s'", + newCtr, err, newCtr, ) + finalErr := healthErr if stopFirst && wasRunning { // Restart the old container only if it was running before we stopped it. oldCtr := fmt.Sprintf("%s/%s", o.OldContainer.ServiceSpec.Name, o.OldContainer.ShortID()) if rollbackErr := cli.StartContainer(ctx, o.ServiceID, o.OldContainer.ID); rollbackErr != nil { - return fmt.Errorf("%w. Rolled back to old container '%s' but failed to restart it: %w", + finalErr = fmt.Errorf("%w. Rolled back to old container '%s' but failed to restart it: %w", healthErr, oldCtr, rollbackErr) + } else { + finalErr = fmt.Errorf("%w. Rolled back to old container '%s'", healthErr, oldCtr) } - return fmt.Errorf("%w. Rolled back to old container '%s'", healthErr, oldCtr) } - return healthErr + return &ContainerHealthError{ + error: finalErr, + ServiceName: o.Spec.Name, + ContainerID: resp.ID, + MachineName: o.MachineName, + } } } diff --git a/pkg/client/deploy/operation/predeploy.go b/pkg/client/deploy/operation/predeploy.go index bace2bbc..702847a7 100644 --- a/pkg/client/deploy/operation/predeploy.go +++ b/pkg/client/deploy/operation/predeploy.go @@ -24,6 +24,10 @@ type PreDeployHookError struct { MachineName string } +func (e *PreDeployHookError) Unwrap() error { + return e.error +} + // DefaultPreDeployTimeout is the maximum duration to wait for a pre-deploy hook container to complete. const DefaultPreDeployTimeout = 5 * time.Minute