feat(rtt): 'machine rtt' command to show round-trip time between macines usign using gossip data (#226)

* Add corrosion admin client function to get RTT to other machines in the cluster

* Add `uc machine rtt` showing all pair-wise RTT stats from corrosion

* Add long description to machine rtt command

* Include machine peer RTTs in InspectMachine instead of adding new gRPC API for it

* Add nil check on InspectMachine's Rtt field to protect from a potential edge case

* Generate docs for machine rtt command

* Handle m.Message being nil when a node is down or unavailble

* Update cli-docs

* Use tui Table instead of tabwriter

* Change reported RTT to the median rather than mean. Still calculate include stddev, as it might be a useful indication of network jitter.

* Show RTT to peers in `wg show` output

* Update cli docs

cmd/uncloud/machine/root.go

@@ -16,6 +16,7 @@ func NewRootCommand() *cobra.Command {
 		NewListCommand(),
 		NewRenameCommand(),
 		NewRmCommand(),
+		NewRTTCommand(),
 		NewUpdateCommand(),
 	)
 	return cmd

cmd/uncloud/machine/rtt.go

@@ -0,0 +1,106 @@
+package machine
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"sort"
+
+	"github.com/psviderski/uncloud/internal/cli"
+	"github.com/psviderski/uncloud/internal/cli/tui"
+	"github.com/spf13/cobra"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+func NewRTTCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "rtt",
+		Short: "Show round-trip times between machines.",
+		Long: `Show round-trip times between machines.
+
+Round-trip time statistics are collected from the Corrosion gossip protocol
+and represent the median of recent RTT samples between each pair of machines
+in the cluster. The values shown include the median RTT and standard deviation
+for each machine-to-machine connection.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			uncli := cmd.Context().Value("cli").(*cli.CLI)
+			return rtt(cmd.Context(), uncli)
+		},
+	}
+	return cmd
+}
+
+func rtt(ctx context.Context, uncli *cli.CLI) error {
+	client, err := uncli.ConnectCluster(ctx)
+	if err != nil {
+		return fmt.Errorf("connect to cluster: %w", err)
+	}
+	defer client.Close()
+
+	// Setup context to proxy request to all machines.
+	ctx, _, err = client.ProxyMachinesContext(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("setup proxy context: %w", err)
+	}
+
+	resp, err := client.MachineClient.InspectMachine(ctx, &emptypb.Empty{})
+	if err != nil {
+		return fmt.Errorf("inspect machines: %w", err)
+	}
+
+	// Map machine IDs to names for display from the response.
+	machineNames := make(map[string]string)
+	for _, m := range resp.Machines {
+		if m.Machine == nil {
+			continue
+		}
+		machineNames[m.Machine.Id] = m.Machine.Name
+	}
+
+	type row struct {
+		machine string
+		peer    string
+		median  float64
+		stdDev  float64
+	}
+	var rows []row
+
+	for _, m := range resp.Machines {
+		// Unlikely to occur, but might be a possible edge case when
+		// a machine is still initializing. So just to be safe..
+		if m.Machine == nil || m.Rtts == nil {
+			continue
+		}
+		for peerID, stats := range m.Rtts {
+			peerName := peerID
+			if name, ok := machineNames[peerID]; ok {
+				peerName = name
+			}
+			rows = append(rows, row{
+				machine: m.Machine.Name,
+				peer:    peerName,
+				median:  stats.Median,
+				stdDev:  stats.StdDev,
+			})
+		}
+	}
+
+	// Sort by machine name then peer name
+	sort.Slice(rows, func(i, j int) bool {
+		if rows[i].machine == rows[j].machine {
+			return rows[i].peer < rows[j].peer
+		}
+		return rows[i].machine < rows[j].machine
+	})
+
+	// Print table
+	t := tui.NewTable()
+	t.Headers("MACHINE", "PEER", "MEDIAN", "STDDEV")
+
+	for _, r := range rows {
+		t.Row(r.machine, r.peer, fmt.Sprintf("%dms", int64(math.Ceil(r.median))), fmt.Sprintf("±%.1fms", r.stdDev))
+	}
+
+	fmt.Println(t)
+	return nil
+}

cmd/uncloud/wg/wg.go

@@ -3,12 +3,14 @@ package wg
 import (
 	"context"
 	"fmt"
+	"math"
 	"strings"
 	"time"
 
 	"github.com/docker/go-units"
 	"github.com/psviderski/uncloud/internal/cli"
 	"github.com/psviderski/uncloud/internal/cli/tui"
+	"github.com/psviderski/uncloud/internal/machine/api/pb"
 	"github.com/spf13/cobra"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	"google.golang.org/grpc/codes"
@@ -74,16 +76,18 @@ func runShow(ctx context.Context, uncli *cli.CLI, opts showOptions) error {
 	if err != nil {
 		return fmt.Errorf("list machines: %w", err)
 	}
-	machinesNamesByPublicKey := make(map[string]string)
+	machinesByPublicKey := make(map[string]*pb.MachineInfo)
 	for _, m := range machines {
 		publicKey := wgtypes.Key(m.Machine.Network.PublicKey).String()
-		machinesNamesByPublicKey[publicKey] = m.Machine.Name
+		machinesByPublicKey[publicKey] = m.Machine
 	}
 
-	// Fetch the machine's name for more descriptive output
+	// Fetch the machine's info and RTTs for display
+	var selfMachine *pb.MachineDetails
 	inspectResp, err := client.MachineClient.InspectMachine(ctx, nil)
 	if err == nil {
-		fmt.Printf("Machine name:         %s\n", inspectResp.Machines[0].Machine.Name)
+		selfMachine = inspectResp.Machines[0]
+		fmt.Printf("Machine name:         %s\n", selfMachine.Machine.Name)
 	}
 
 	fmt.Printf("WireGuard interface:  %s\n", resp.InterfaceName)
@@ -97,12 +101,19 @@ func runShow(ctx context.Context, uncli *cli.CLI, opts showOptions) error {
 	}
 
 	t := tui.NewTable()
-	t.Headers("PEER", "PUBLIC KEY", "ENDPOINT", "HANDSHAKE", "RECEIVED", "SENT", "ALLOWED IPS")
+	t.Headers("PEER", "PUBLIC KEY", "ENDPOINT", "HANDSHAKE", "RTT", "RECEIVED", "SENT", "ALLOWED IPS")
 
 	for _, peer := range resp.Peers {
-		machineName, ok := machinesNamesByPublicKey[wgtypes.Key(peer.PublicKey).String()]
-		if !ok {
-			machineName = "(unknown)"
+		publicKeyStr := wgtypes.Key(peer.PublicKey).String()
+		machineName := "(unknown)"
+		rtt := "-"
+		if m, ok := machinesByPublicKey[publicKeyStr]; ok {
+			machineName = m.Name
+			if selfMachine != nil {
+				if stats, ok := selfMachine.Rtts[m.Id]; ok {
+					rtt = fmt.Sprintf("%dms", int64(math.Ceil(stats.Median)))
+				}
+			}
 		}
 
 		lastHandshake := ""
@@ -112,9 +123,10 @@ func runShow(ctx context.Context, uncli *cli.CLI, opts showOptions) error {
 
 		t.Row(
 			machineName,
-			wgtypes.Key(peer.PublicKey).String(),
+			publicKeyStr,
 			peer.Endpoint,
 			lastHandshake,
+			rtt,
 			units.HumanSize(float64(peer.ReceiveBytes)),
 			units.HumanSize(float64(peer.TransmitBytes)),
 			strings.Join(peer.AllowedIps, tui.Faint.Render(", ")),