diff --git a/cmd/uncloud/machine/root.go b/cmd/uncloud/machine/root.go index cf42c026..1cf8156b 100644 --- a/cmd/uncloud/machine/root.go +++ b/cmd/uncloud/machine/root.go @@ -16,6 +16,7 @@ func NewRootCommand() *cobra.Command { NewListCommand(), NewRenameCommand(), NewRmCommand(), + NewRTTCommand(), NewUpdateCommand(), ) return cmd diff --git a/cmd/uncloud/machine/rtt.go b/cmd/uncloud/machine/rtt.go new file mode 100644 index 00000000..02616e85 --- /dev/null +++ b/cmd/uncloud/machine/rtt.go @@ -0,0 +1,106 @@ +package machine + +import ( + "context" + "fmt" + "math" + "sort" + + "github.com/psviderski/uncloud/internal/cli" + "github.com/psviderski/uncloud/internal/cli/tui" + "github.com/spf13/cobra" + "google.golang.org/protobuf/types/known/emptypb" +) + +func NewRTTCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "rtt", + Short: "Show round-trip times between machines.", + Long: `Show round-trip times between machines. + +Round-trip time statistics are collected from the Corrosion gossip protocol +and represent the median of recent RTT samples between each pair of machines +in the cluster. The values shown include the median RTT and standard deviation +for each machine-to-machine connection.`, + RunE: func(cmd *cobra.Command, args []string) error { + uncli := cmd.Context().Value("cli").(*cli.CLI) + return rtt(cmd.Context(), uncli) + }, + } + return cmd +} + +func rtt(ctx context.Context, uncli *cli.CLI) error { + client, err := uncli.ConnectCluster(ctx) + if err != nil { + return fmt.Errorf("connect to cluster: %w", err) + } + defer client.Close() + + // Setup context to proxy request to all machines. + ctx, _, err = client.ProxyMachinesContext(ctx, nil) + if err != nil { + return fmt.Errorf("setup proxy context: %w", err) + } + + resp, err := client.MachineClient.InspectMachine(ctx, &emptypb.Empty{}) + if err != nil { + return fmt.Errorf("inspect machines: %w", err) + } + + // Map machine IDs to names for display from the response. + machineNames := make(map[string]string) + for _, m := range resp.Machines { + if m.Machine == nil { + continue + } + machineNames[m.Machine.Id] = m.Machine.Name + } + + type row struct { + machine string + peer string + median float64 + stdDev float64 + } + var rows []row + + for _, m := range resp.Machines { + // Unlikely to occur, but might be a possible edge case when + // a machine is still initializing. So just to be safe.. + if m.Machine == nil || m.Rtts == nil { + continue + } + for peerID, stats := range m.Rtts { + peerName := peerID + if name, ok := machineNames[peerID]; ok { + peerName = name + } + rows = append(rows, row{ + machine: m.Machine.Name, + peer: peerName, + median: stats.Median, + stdDev: stats.StdDev, + }) + } + } + + // Sort by machine name then peer name + sort.Slice(rows, func(i, j int) bool { + if rows[i].machine == rows[j].machine { + return rows[i].peer < rows[j].peer + } + return rows[i].machine < rows[j].machine + }) + + // Print table + t := tui.NewTable() + t.Headers("MACHINE", "PEER", "MEDIAN", "STDDEV") + + for _, r := range rows { + t.Row(r.machine, r.peer, fmt.Sprintf("%dms", int64(math.Ceil(r.median))), fmt.Sprintf("±%.1fms", r.stdDev)) + } + + fmt.Println(t) + return nil +} diff --git a/cmd/uncloud/wg/wg.go b/cmd/uncloud/wg/wg.go index 4a8b99b7..f68ca2c3 100644 --- a/cmd/uncloud/wg/wg.go +++ b/cmd/uncloud/wg/wg.go @@ -3,12 +3,14 @@ package wg import ( "context" "fmt" + "math" "strings" "time" "github.com/docker/go-units" "github.com/psviderski/uncloud/internal/cli" "github.com/psviderski/uncloud/internal/cli/tui" + "github.com/psviderski/uncloud/internal/machine/api/pb" "github.com/spf13/cobra" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "google.golang.org/grpc/codes" @@ -74,16 +76,18 @@ func runShow(ctx context.Context, uncli *cli.CLI, opts showOptions) error { if err != nil { return fmt.Errorf("list machines: %w", err) } - machinesNamesByPublicKey := make(map[string]string) + machinesByPublicKey := make(map[string]*pb.MachineInfo) for _, m := range machines { publicKey := wgtypes.Key(m.Machine.Network.PublicKey).String() - machinesNamesByPublicKey[publicKey] = m.Machine.Name + machinesByPublicKey[publicKey] = m.Machine } - // Fetch the machine's name for more descriptive output + // Fetch the machine's info and RTTs for display + var selfMachine *pb.MachineDetails inspectResp, err := client.MachineClient.InspectMachine(ctx, nil) if err == nil { - fmt.Printf("Machine name: %s\n", inspectResp.Machines[0].Machine.Name) + selfMachine = inspectResp.Machines[0] + fmt.Printf("Machine name: %s\n", selfMachine.Machine.Name) } fmt.Printf("WireGuard interface: %s\n", resp.InterfaceName) @@ -97,12 +101,19 @@ func runShow(ctx context.Context, uncli *cli.CLI, opts showOptions) error { } t := tui.NewTable() - t.Headers("PEER", "PUBLIC KEY", "ENDPOINT", "HANDSHAKE", "RECEIVED", "SENT", "ALLOWED IPS") + t.Headers("PEER", "PUBLIC KEY", "ENDPOINT", "HANDSHAKE", "RTT", "RECEIVED", "SENT", "ALLOWED IPS") for _, peer := range resp.Peers { - machineName, ok := machinesNamesByPublicKey[wgtypes.Key(peer.PublicKey).String()] - if !ok { - machineName = "(unknown)" + publicKeyStr := wgtypes.Key(peer.PublicKey).String() + machineName := "(unknown)" + rtt := "-" + if m, ok := machinesByPublicKey[publicKeyStr]; ok { + machineName = m.Name + if selfMachine != nil { + if stats, ok := selfMachine.Rtts[m.Id]; ok { + rtt = fmt.Sprintf("%dms", int64(math.Ceil(stats.Median))) + } + } } lastHandshake := "" @@ -112,9 +123,10 @@ func runShow(ctx context.Context, uncli *cli.CLI, opts showOptions) error { t.Row( machineName, - wgtypes.Key(peer.PublicKey).String(), + publicKeyStr, peer.Endpoint, lastHandshake, + rtt, units.HumanSize(float64(peer.ReceiveBytes)), units.HumanSize(float64(peer.TransmitBytes)), strings.Join(peer.AllowedIps, tui.Faint.Render(", ")), diff --git a/internal/corrosion/admin.go b/internal/corrosion/admin.go index 660e2534..f2733f5f 100644 --- a/internal/corrosion/admin.go +++ b/internal/corrosion/admin.go @@ -6,8 +6,10 @@ import ( "errors" "fmt" "io" + "math" "net" "net/netip" + "sort" "time" ) @@ -256,3 +258,106 @@ func (c *AdminClient) ClusterMembershipStates(latest bool) ([]ClusterMembershipS } return states, parseErr } + +type MemberRTTStats struct { + Addr netip.AddrPort + Median float64 + StdDev float64 +} + +// ClusterMemberRTTs returns the median and standard deviation of round-trip times to each cluster member. +func (c *AdminClient) ClusterMemberRTTs() ([]MemberRTTStats, error) { + respCh, err := c.SendCommand([]byte("{\"Cluster\":\"Members\"}")) + if err != nil { + return nil, err + } + + var stats []MemberRTTStats + var parseErr error + + for r := range respCh { + if r.Err != nil { + return nil, r.Err + } + + addr, rtts, err := parseClusterMemberRTT(r.JSON) + if err != nil { + parseErr = errors.Join(parseErr, err) + continue + } + + if len(rtts) == 0 { + continue + } + + sort.Float64s(rtts) + n := len(rtts) + var median float64 + if n%2 == 0 { + median = (rtts[n/2-1] + rtts[n/2]) / 2 + } else { + median = rtts[n/2] + } + + var sum float64 + for _, rtt := range rtts { + sum += rtt + } + avg := sum / float64(n) + + var varianceSum float64 + for _, rtt := range rtts { + diff := rtt - avg + varianceSum += diff * diff + } + stdDev := math.Sqrt(varianceSum / float64(n)) + + stats = append(stats, MemberRTTStats{ + Addr: addr, + Median: median, + StdDev: stdDev, + }) + } + + return stats, parseErr +} + +func parseClusterMemberRTT(json map[string]any) (netip.AddrPort, []float64, error) { + var addr netip.AddrPort + var rtts []float64 + var err error + + // Parse state to get Addr + stateObj, ok := json["state"].(map[string]any) + if !ok { + return addr, nil, fmt.Errorf("missing or invalid 'state' field") + } + + if addrStr, ok := stateObj["addr"].(string); ok { + addr, err = netip.ParseAddrPort(addrStr) + if err != nil { + return addr, nil, fmt.Errorf("parse 'addr' field: %w", err) + } + } else { + return addr, nil, fmt.Errorf("missing or invalid 'addr' field in 'state'") + } + + // Parse RTTs + if rttsVal, ok := json["rtts"]; ok { + if rttsSlice, ok := rttsVal.([]any); ok { + for _, v := range rttsSlice { + if f, ok := v.(float64); ok { + rtts = append(rtts, f) + } else { + return addr, nil, fmt.Errorf("invalid rtt value type: %T", v) + } + } + } else { + return addr, nil, fmt.Errorf("invalid 'rtts' field type: %T", rttsVal) + } + } else { + return addr, nil, fmt.Errorf("missing 'rtts' field") + } + + return addr, rtts, nil +} diff --git a/internal/machine/api/pb/machine.pb.go b/internal/machine/api/pb/machine.pb.go index 37d3031b..f8f95205 100644 --- a/internal/machine/api/pb/machine.pb.go +++ b/internal/machine/api/pb/machine.pb.go @@ -495,6 +495,8 @@ type MachineDetails struct { Machine *MachineInfo `protobuf:"bytes,2,opt,name=machine,proto3" json:"machine,omitempty"` // Current Corrosion cr-sqlite database version (Lamport timestamp) of the cluster store. StoreDbVersion int64 `protobuf:"varint,3,opt,name=store_db_version,json=storeDbVersion,proto3" json:"store_db_version,omitempty"` + // Round-trip times to other machines in the cluster, keyed by peer machine ID. + Rtts map[string]*RTTStats `protobuf:"bytes,4,rep,name=rtts,proto3" json:"rtts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *MachineDetails) Reset() { @@ -550,6 +552,13 @@ func (x *MachineDetails) GetStoreDbVersion() int64 { return 0 } +func (x *MachineDetails) GetRtts() map[string]*RTTStats { + if x != nil { + return x.Rtts + } + return nil +} + type TokenResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -958,6 +967,61 @@ func (x *WireGuardPeer) GetAllowedIps() []string { return nil } +type RTTStats struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Median float64 `protobuf:"fixed64,1,opt,name=median,proto3" json:"median,omitempty"` + StdDev float64 `protobuf:"fixed64,2,opt,name=std_dev,json=stdDev,proto3" json:"std_dev,omitempty"` +} + +func (x *RTTStats) Reset() { + *x = RTTStats{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_machine_api_pb_machine_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RTTStats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RTTStats) ProtoMessage() {} + +func (x *RTTStats) ProtoReflect() protoreflect.Message { + mi := &file_internal_machine_api_pb_machine_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RTTStats.ProtoReflect.Descriptor instead. +func (*RTTStats) Descriptor() ([]byte, []int) { + return file_internal_machine_api_pb_machine_proto_rawDescGZIP(), []int{15} +} + +func (x *RTTStats) GetMedian() float64 { + if x != nil { + return x.Median + } + return 0 +} + +func (x *RTTStats) GetStdDev() float64 { + if x != nil { + return x.StdDev + } + return 0 +} + type Service_Container struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -971,7 +1035,7 @@ type Service_Container struct { func (x *Service_Container) Reset() { *x = Service_Container{} if protoimpl.UnsafeEnabled { - mi := &file_internal_machine_api_pb_machine_proto_msgTypes[15] + mi := &file_internal_machine_api_pb_machine_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -984,7 +1048,7 @@ func (x *Service_Container) String() string { func (*Service_Container) ProtoMessage() {} func (x *Service_Container) ProtoReflect() protoreflect.Message { - mi := &file_internal_machine_api_pb_machine_proto_msgTypes[15] + mi := &file_internal_machine_api_pb_machine_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1086,7 +1150,7 @@ var file_internal_machine_api_pb_machine_proto_rawDesc = []byte{ 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, - 0x52, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x22, 0x91, 0x01, 0x0a, 0x0e, 0x4d, + 0x52, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x22, 0x8c, 0x02, 0x0a, 0x0e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x29, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, @@ -1095,103 +1159,115 @@ var file_internal_machine_api_pb_machine_proto_rawDesc = []byte{ 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x64, 0x62, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x44, 0x62, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x25, - 0x0a, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x0e, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x65, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xc3, 0x01, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, - 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x36, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, - 0x61, 0x70, 0x69, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x73, 0x1a, 0x48, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1d, - 0x0a, 0x0a, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x1c, 0x0a, - 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x22, 0x27, 0x0a, 0x15, 0x49, - 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x69, 0x64, 0x22, 0x40, 0x0a, 0x16, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, - 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x07, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0xb2, 0x01, 0x0a, 0x1f, 0x49, 0x6e, 0x73, 0x70, 0x65, - 0x63, 0x74, 0x57, 0x69, 0x72, 0x65, 0x47, 0x75, 0x61, 0x72, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, - 0x72, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, - 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x50, 0x6f, 0x72, - 0x74, 0x12, 0x28, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x57, 0x69, 0x72, 0x65, 0x47, 0x75, 0x61, 0x72, 0x64, - 0x50, 0x65, 0x65, 0x72, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x22, 0x83, 0x02, 0x0a, 0x0d, - 0x57, 0x69, 0x72, 0x65, 0x47, 0x75, 0x61, 0x72, 0x64, 0x50, 0x65, 0x65, 0x72, 0x12, 0x1d, 0x0a, - 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1a, 0x0a, 0x08, - 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x4a, 0x0a, 0x13, 0x6c, 0x61, 0x73, 0x74, - 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, - 0x54, 0x69, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x5f, - 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x72, 0x65, 0x63, - 0x65, 0x69, 0x76, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x72, 0x61, - 0x6e, 0x73, 0x6d, 0x69, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x0d, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6d, 0x69, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, - 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x69, 0x70, 0x73, 0x18, - 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70, - 0x73, 0x32, 0x95, 0x05, 0x0a, 0x07, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x4d, 0x0a, - 0x12, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x50, 0x72, 0x65, 0x72, 0x65, 0x71, 0x75, 0x69, 0x73, 0x69, - 0x74, 0x65, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x50, 0x72, 0x65, 0x72, 0x65, 0x71, 0x75, 0x69, 0x73, - 0x69, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x0b, - 0x49, 0x6e, 0x69, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x17, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x43, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, - 0x0a, 0x0b, 0x4a, 0x6f, 0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x17, 0x2e, - 0x61, 0x70, 0x69, 0x2e, 0x4a, 0x6f, 0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x33, - 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, - 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x07, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x12, 0x16, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x44, 0x62, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x31, + 0x0a, 0x04, 0x72, 0x74, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x73, 0x2e, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x72, 0x74, 0x74, + 0x73, 0x1a, 0x46, 0x0a, 0x09, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x23, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x54, 0x54, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25, 0x0a, 0x0d, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, + 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, + 0x22, 0x0e, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x22, 0xc3, 0x01, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x36, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x1a, 0x48, 0x0a, 0x09, + 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x61, 0x63, + 0x68, 0x69, 0x6e, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, + 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x22, 0x27, 0x0a, 0x15, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, + 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, + 0x40, 0x0a, 0x16, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x07, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x22, 0xb2, 0x01, 0x0a, 0x1f, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x57, 0x69, 0x72, + 0x65, 0x47, 0x75, 0x61, 0x72, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, + 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, + 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, + 0x69, 0x73, 0x74, 0x65, 0x6e, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x0a, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x28, 0x0a, 0x05, + 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x57, 0x69, 0x72, 0x65, 0x47, 0x75, 0x61, 0x72, 0x64, 0x50, 0x65, 0x65, 0x72, 0x52, + 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x22, 0x83, 0x02, 0x0a, 0x0d, 0x57, 0x69, 0x72, 0x65, 0x47, + 0x75, 0x61, 0x72, 0x64, 0x50, 0x65, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, + 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x70, 0x75, + 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x12, 0x4a, 0x0a, 0x13, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x68, 0x61, 0x6e, 0x64, + 0x73, 0x68, 0x61, 0x6b, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x11, 0x6c, 0x61, + 0x73, 0x74, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, + 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6d, 0x69, 0x74, + 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x6d, 0x69, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x61, + 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x69, 0x70, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70, 0x73, 0x22, 0x3b, 0x0a, 0x08, + 0x52, 0x54, 0x54, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x64, 0x69, + 0x61, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x06, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x6e, + 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x64, 0x5f, 0x64, 0x65, 0x76, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x06, 0x73, 0x74, 0x64, 0x44, 0x65, 0x76, 0x32, 0x95, 0x05, 0x0a, 0x07, 0x4d, 0x61, + 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x4d, 0x0a, 0x12, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x50, 0x72, + 0x65, 0x72, 0x65, 0x71, 0x75, 0x69, 0x73, 0x69, 0x74, 0x65, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x50, + 0x72, 0x65, 0x72, 0x65, 0x71, 0x75, 0x69, 0x73, 0x69, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x0b, 0x49, 0x6e, 0x69, 0x74, 0x43, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x12, 0x17, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x43, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0b, 0x4a, 0x6f, 0x69, 0x6e, 0x43, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x17, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4a, 0x6f, 0x69, 0x6e, + 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x45, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x70, - 0x65, 0x63, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, - 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x57, 0x0a, 0x17, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x57, 0x69, 0x72, 0x65, 0x47, 0x75, - 0x61, 0x72, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x1a, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, - 0x57, 0x69, 0x72, 0x65, 0x47, 0x75, 0x61, 0x72, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x05, 0x52, 0x65, 0x73, 0x65, - 0x74, 0x12, 0x11, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x49, 0x0a, 0x0e, - 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x1a, - 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x0b, 0x4d, 0x61, 0x63, 0x68, 0x69, - 0x6e, 0x65, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x6f, 0x67, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, - 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x30, 0x01, 0x42, 0x37, 0x5a, 0x35, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, - 0x6b, 0x69, 0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, - 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x33, 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x07, 0x49, + 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x10, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, + 0x12, 0x45, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, + 0x6e, 0x65, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x17, 0x49, 0x6e, 0x73, 0x70, 0x65, + 0x63, 0x74, 0x57, 0x69, 0x72, 0x65, 0x47, 0x75, 0x61, 0x72, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x24, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x57, 0x69, 0x72, 0x65, 0x47, 0x75, 0x61, 0x72, + 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x32, 0x0a, 0x05, 0x52, 0x65, 0x73, 0x65, 0x74, 0x12, 0x11, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x52, 0x65, 0x73, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x12, 0x49, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x6e, 0x73, + 0x70, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x30, 0x0a, 0x0b, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x10, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x0d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x30, + 0x01, 0x42, 0x37, 0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x6b, 0x69, 0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6d, 0x61, 0x63, 0x68, + 0x69, 0x6e, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -1206,7 +1282,7 @@ func file_internal_machine_api_pb_machine_proto_rawDescGZIP() []byte { return file_internal_machine_api_pb_machine_proto_rawDescData } -var file_internal_machine_api_pb_machine_proto_msgTypes = make([]protoimpl.MessageInfo, 16) +var file_internal_machine_api_pb_machine_proto_msgTypes = make([]protoimpl.MessageInfo, 18) var file_internal_machine_api_pb_machine_proto_goTypes = []any{ (*MachineInfo)(nil), // 0: api.MachineInfo (*NetworkConfig)(nil), // 1: api.NetworkConfig @@ -1223,60 +1299,64 @@ var file_internal_machine_api_pb_machine_proto_goTypes = []any{ (*InspectServiceResponse)(nil), // 12: api.InspectServiceResponse (*InspectWireGuardNetworkResponse)(nil), // 13: api.InspectWireGuardNetworkResponse (*WireGuardPeer)(nil), // 14: api.WireGuardPeer - (*Service_Container)(nil), // 15: api.Service.Container - (*IP)(nil), // 16: api.IP - (*IPPrefix)(nil), // 17: api.IPPrefix - (*IPPort)(nil), // 18: api.IPPort - (*Metadata)(nil), // 19: api.Metadata - (*timestamppb.Timestamp)(nil), // 20: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 21: google.protobuf.Empty - (*LogsRequest)(nil), // 22: api.LogsRequest - (*LogEntry)(nil), // 23: api.LogEntry + (*RTTStats)(nil), // 15: api.RTTStats + nil, // 16: api.MachineDetails.RttsEntry + (*Service_Container)(nil), // 17: api.Service.Container + (*IP)(nil), // 18: api.IP + (*IPPrefix)(nil), // 19: api.IPPrefix + (*IPPort)(nil), // 20: api.IPPort + (*Metadata)(nil), // 21: api.Metadata + (*timestamppb.Timestamp)(nil), // 22: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 23: google.protobuf.Empty + (*LogsRequest)(nil), // 24: api.LogsRequest + (*LogEntry)(nil), // 25: api.LogEntry } var file_internal_machine_api_pb_machine_proto_depIdxs = []int32{ 1, // 0: api.MachineInfo.network:type_name -> api.NetworkConfig - 16, // 1: api.MachineInfo.public_ip:type_name -> api.IP - 17, // 2: api.NetworkConfig.subnet:type_name -> api.IPPrefix - 16, // 3: api.NetworkConfig.management_ip:type_name -> api.IP - 18, // 4: api.NetworkConfig.endpoints:type_name -> api.IPPort - 17, // 5: api.InitClusterRequest.network:type_name -> api.IPPrefix - 16, // 6: api.InitClusterRequest.public_ip:type_name -> api.IP - 18, // 7: api.InitClusterRequest.wireguard_endpoints:type_name -> api.IPPort + 18, // 1: api.MachineInfo.public_ip:type_name -> api.IP + 19, // 2: api.NetworkConfig.subnet:type_name -> api.IPPrefix + 18, // 3: api.NetworkConfig.management_ip:type_name -> api.IP + 20, // 4: api.NetworkConfig.endpoints:type_name -> api.IPPort + 19, // 5: api.InitClusterRequest.network:type_name -> api.IPPrefix + 18, // 6: api.InitClusterRequest.public_ip:type_name -> api.IP + 20, // 7: api.InitClusterRequest.wireguard_endpoints:type_name -> api.IPPort 0, // 8: api.InitClusterResponse.machine:type_name -> api.MachineInfo 0, // 9: api.JoinClusterRequest.machine:type_name -> api.MachineInfo 0, // 10: api.JoinClusterRequest.other_machines:type_name -> api.MachineInfo 7, // 11: api.InspectMachineResponse.machines:type_name -> api.MachineDetails - 19, // 12: api.MachineDetails.metadata:type_name -> api.Metadata + 21, // 12: api.MachineDetails.metadata:type_name -> api.Metadata 0, // 13: api.MachineDetails.machine:type_name -> api.MachineInfo - 15, // 14: api.Service.containers:type_name -> api.Service.Container - 10, // 15: api.InspectServiceResponse.service:type_name -> api.Service - 14, // 16: api.InspectWireGuardNetworkResponse.peers:type_name -> api.WireGuardPeer - 20, // 17: api.WireGuardPeer.last_handshake_time:type_name -> google.protobuf.Timestamp - 21, // 18: api.Machine.CheckPrerequisites:input_type -> google.protobuf.Empty - 3, // 19: api.Machine.InitCluster:input_type -> api.InitClusterRequest - 5, // 20: api.Machine.JoinCluster:input_type -> api.JoinClusterRequest - 21, // 21: api.Machine.Token:input_type -> google.protobuf.Empty - 21, // 22: api.Machine.Inspect:input_type -> google.protobuf.Empty - 21, // 23: api.Machine.InspectMachine:input_type -> google.protobuf.Empty - 21, // 24: api.Machine.InspectWireGuardNetwork:input_type -> google.protobuf.Empty - 9, // 25: api.Machine.Reset:input_type -> api.ResetRequest - 11, // 26: api.Machine.InspectService:input_type -> api.InspectServiceRequest - 22, // 27: api.Machine.MachineLogs:input_type -> api.LogsRequest - 2, // 28: api.Machine.CheckPrerequisites:output_type -> api.CheckPrerequisitesResponse - 4, // 29: api.Machine.InitCluster:output_type -> api.InitClusterResponse - 21, // 30: api.Machine.JoinCluster:output_type -> google.protobuf.Empty - 8, // 31: api.Machine.Token:output_type -> api.TokenResponse - 0, // 32: api.Machine.Inspect:output_type -> api.MachineInfo - 6, // 33: api.Machine.InspectMachine:output_type -> api.InspectMachineResponse - 13, // 34: api.Machine.InspectWireGuardNetwork:output_type -> api.InspectWireGuardNetworkResponse - 21, // 35: api.Machine.Reset:output_type -> google.protobuf.Empty - 12, // 36: api.Machine.InspectService:output_type -> api.InspectServiceResponse - 23, // 37: api.Machine.MachineLogs:output_type -> api.LogEntry - 28, // [28:38] is the sub-list for method output_type - 18, // [18:28] is the sub-list for method input_type - 18, // [18:18] is the sub-list for extension type_name - 18, // [18:18] is the sub-list for extension extendee - 0, // [0:18] is the sub-list for field type_name + 16, // 14: api.MachineDetails.rtts:type_name -> api.MachineDetails.RttsEntry + 17, // 15: api.Service.containers:type_name -> api.Service.Container + 10, // 16: api.InspectServiceResponse.service:type_name -> api.Service + 14, // 17: api.InspectWireGuardNetworkResponse.peers:type_name -> api.WireGuardPeer + 22, // 18: api.WireGuardPeer.last_handshake_time:type_name -> google.protobuf.Timestamp + 15, // 19: api.MachineDetails.RttsEntry.value:type_name -> api.RTTStats + 23, // 20: api.Machine.CheckPrerequisites:input_type -> google.protobuf.Empty + 3, // 21: api.Machine.InitCluster:input_type -> api.InitClusterRequest + 5, // 22: api.Machine.JoinCluster:input_type -> api.JoinClusterRequest + 23, // 23: api.Machine.Token:input_type -> google.protobuf.Empty + 23, // 24: api.Machine.Inspect:input_type -> google.protobuf.Empty + 23, // 25: api.Machine.InspectMachine:input_type -> google.protobuf.Empty + 23, // 26: api.Machine.InspectWireGuardNetwork:input_type -> google.protobuf.Empty + 9, // 27: api.Machine.Reset:input_type -> api.ResetRequest + 11, // 28: api.Machine.InspectService:input_type -> api.InspectServiceRequest + 24, // 29: api.Machine.MachineLogs:input_type -> api.LogsRequest + 2, // 30: api.Machine.CheckPrerequisites:output_type -> api.CheckPrerequisitesResponse + 4, // 31: api.Machine.InitCluster:output_type -> api.InitClusterResponse + 23, // 32: api.Machine.JoinCluster:output_type -> google.protobuf.Empty + 8, // 33: api.Machine.Token:output_type -> api.TokenResponse + 0, // 34: api.Machine.Inspect:output_type -> api.MachineInfo + 6, // 35: api.Machine.InspectMachine:output_type -> api.InspectMachineResponse + 13, // 36: api.Machine.InspectWireGuardNetwork:output_type -> api.InspectWireGuardNetworkResponse + 23, // 37: api.Machine.Reset:output_type -> google.protobuf.Empty + 12, // 38: api.Machine.InspectService:output_type -> api.InspectServiceResponse + 25, // 39: api.Machine.MachineLogs:output_type -> api.LogEntry + 30, // [30:40] is the sub-list for method output_type + 20, // [20:30] is the sub-list for method input_type + 20, // [20:20] is the sub-list for extension type_name + 20, // [20:20] is the sub-list for extension extendee + 0, // [0:20] is the sub-list for field type_name } func init() { file_internal_machine_api_pb_machine_proto_init() } @@ -1467,6 +1547,18 @@ func file_internal_machine_api_pb_machine_proto_init() { } } file_internal_machine_api_pb_machine_proto_msgTypes[15].Exporter = func(v any, i int) any { + switch v := v.(*RTTStats); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_machine_api_pb_machine_proto_msgTypes[17].Exporter = func(v any, i int) any { switch v := v.(*Service_Container); i { case 0: return &v.state @@ -1489,7 +1581,7 @@ func file_internal_machine_api_pb_machine_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_internal_machine_api_pb_machine_proto_rawDesc, NumEnums: 0, - NumMessages: 16, + NumMessages: 18, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/machine/api/pb/machine.proto b/internal/machine/api/pb/machine.proto index 737386c4..e4346bfc 100644 --- a/internal/machine/api/pb/machine.proto +++ b/internal/machine/api/pb/machine.proto @@ -83,6 +83,8 @@ message MachineDetails { MachineInfo machine = 2; // Current Corrosion cr-sqlite database version (Lamport timestamp) of the cluster store. int64 store_db_version = 3; + // Round-trip times to other machines in the cluster, keyed by peer machine ID. + map rtts = 4; } message TokenResponse { @@ -129,3 +131,8 @@ message WireGuardPeer { int64 transmit_bytes = 5; repeated string allowed_ips = 6; } + +message RTTStats { + double median = 1; + double std_dev = 2; +} diff --git a/internal/machine/cluster/cluster.go b/internal/machine/cluster/cluster.go index d2cc4143..f7782d26 100644 --- a/internal/machine/cluster/cluster.go +++ b/internal/machine/cluster/cluster.go @@ -344,3 +344,8 @@ func (c *Cluster) RemoveMachine(ctx context.Context, req *pb.RemoveMachineReques return &emptypb.Empty{}, nil } + +// MemberRTTs returns the average and standard deviation of round-trip times from this member to each cluster member. +func (c *Cluster) MemberRTTs() ([]corrosion.MemberRTTStats, error) { + return c.corroAdmin.ClusterMemberRTTs() +} diff --git a/internal/machine/machine.go b/internal/machine/machine.go index 28c66ba1..90d789c7 100644 --- a/internal/machine/machine.go +++ b/internal/machine/machine.go @@ -912,6 +912,14 @@ func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.Ins return nil, status.Errorf(codes.Internal, "get database version of the cluster store: %v", err) } + var rtts map[string]*pb.RTTStats + if m.Initialised() { + rtts, err = m.getMachineRTTs(ctx) + if err != nil { + return nil, err + } + } + return &pb.InspectMachineResponse{ Machines: []*pb.MachineDetails{ { @@ -926,11 +934,46 @@ func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.Ins }, }, StoreDbVersion: dbVersion, + Rtts: rtts, }, }, }, nil } +// getMachineRTTs retrieves round-trip times to other machines in the cluster. +func (m *Machine) getMachineRTTs(ctx context.Context) (map[string]*pb.RTTStats, error) { + rtts, err := m.cluster.MemberRTTs() + if err != nil { + return nil, status.Errorf(codes.Internal, "get member rtts: %v", err) + } + + // List machines to map IPs to Machine IDs. + machines, err := m.store.ListMachines(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "list machines: %v", err) + } + + // Map Management IP -> Machine ID + ipToMachineID := make(map[netip.Addr]string) + for _, mach := range machines { + ip, _ := mach.Network.ManagementIp.ToAddr() + ipToMachineID[ip] = mach.Id + } + + pbRTTs := make(map[string]*pb.RTTStats) + for _, stats := range rtts { + // Corrosion uses the management IP for gossip. + if mid, ok := ipToMachineID[stats.Addr.Addr()]; ok { + pbRTTs[mid] = &pb.RTTStats{ + Median: stats.Median, + StdDev: stats.StdDev, + } + } + } + + return pbRTTs, nil +} + // IsNetworkReady returns true if the Docker network is ready for containers. func (m *Machine) IsNetworkReady() bool { if !m.Initialised() { diff --git a/website/docs/9-cli-reference/uc_machine.md b/website/docs/9-cli-reference/uc_machine.md index 70b08a05..0a777ed3 100644 --- a/website/docs/9-cli-reference/uc_machine.md +++ b/website/docs/9-cli-reference/uc_machine.md @@ -25,5 +25,6 @@ Manage machines in the cluster. * [uc machine ls](uc_machine_ls.md) - List machines in a cluster. * [uc machine rename](uc_machine_rename.md) - Rename a machine in the cluster. * [uc machine rm](uc_machine_rm.md) - Remove a machine from a cluster and reset it. +* [uc machine rtt](uc_machine_rtt.md) - Show round-trip times between machines. * [uc machine update](uc_machine_update.md) - Update machine configuration in the cluster. diff --git a/website/docs/9-cli-reference/uc_machine_rtt.md b/website/docs/9-cli-reference/uc_machine_rtt.md new file mode 100644 index 00000000..999c4b41 --- /dev/null +++ b/website/docs/9-cli-reference/uc_machine_rtt.md @@ -0,0 +1,36 @@ +# uc machine rtt + +Show round-trip times between machines. + +## Synopsis + +Show round-trip times between machines. + +Round-trip time statistics are collected from the Corrosion gossip protocol +and represent the median of recent RTT samples between each pair of machines +in the cluster. The values shown include the median RTT and standard deviation +for each machine-to-machine connection. + +``` +uc machine rtt [flags] +``` + +## Options + +``` + -h, --help help for rtt +``` + +## Options inherited from parent commands + +``` + --connect string Connect to a remote cluster machine without using the Uncloud configuration file. [$UNCLOUD_CONNECT] + Format: [ssh://]user@host[:port], ssh+go://user@host[:port], tcp://host:port, or unix:///path/to/uncloud.sock + -c, --context string Name of the cluster context to use (default is the current context). [$UNCLOUD_CONTEXT] + --uncloud-config string Path to the Uncloud configuration file. [$UNCLOUD_CONFIG] (default "~/.config/uncloud/config.yaml") +``` + +## See also + +* [uc machine](uc_machine.md) - Manage machines in the cluster. +