From 9236aa341a73770ed27acd3522d34e22eaf2cfcb Mon Sep 17 00:00:00 2001 From: Pasha Sviderski Date: Mon, 25 May 2026 14:09:51 +1000 Subject: [PATCH] refactor: migrate to latest Corrosion version 2026.5.14 for ucind (compare versions vector for store sync) --- internal/cli/cli.go | 14 +- internal/machine/api/pb/machine.pb.go | 229 ++++++++++++++------------ internal/machine/api/pb/machine.proto | 10 +- internal/machine/cluster.go | 118 +++++++------ internal/machine/machine.go | 10 +- internal/machine/state.go | 7 +- internal/machine/store/store.go | 33 ++-- internal/ucind/cluster.go | 6 +- 8 files changed, 237 insertions(+), 190 deletions(-) diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 2ad07129..d28e8068 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -434,8 +434,8 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client return nil, nil, fmt.Errorf("add machine to cluster (context '%s'): %w", contextName, err) } - // Get the current store DB version from the cluster to pass to the join request. - var storeDBVersion int64 + // Snapshot the cluster store version so the new machine can catch up before participating. + var storeVersion map[string]int64 inspectResp, err := c.MachineClient.InspectMachine(ctx, &emptypb.Empty{}) if err != nil { // TODO(lhf): remove Unimplemented check when v0.17.0 is released. @@ -443,7 +443,7 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client return nil, nil, fmt.Errorf("inspect current cluster machine: %w", err) } } else { - storeDBVersion = inspectResp.Machines[0].StoreDbVersion + storeVersion = inspectResp.Machines[0].StoreVersion } // Get the most up-to-date list of other machines in the cluster to include them in the join request. @@ -460,10 +460,10 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client // Configure the remote machine to join the cluster. joinReq := &pb.JoinClusterRequest{ - Machine: addResp.Machine, - OtherMachines: otherMachines, - MinStoreDbVersion: storeDBVersion, - WireguardPort: int32(opts.WireguardPort), + Machine: addResp.Machine, + OtherMachines: otherMachines, + MinStoreVersion: storeVersion, + WireguardPort: int32(opts.WireguardPort), } if _, err = machineClient.JoinCluster(ctx, joinReq); err != nil { return nil, nil, fmt.Errorf("join cluster: %w", err) diff --git a/internal/machine/api/pb/machine.pb.go b/internal/machine/api/pb/machine.pb.go index 76e2eb4b..144df41c 100644 --- a/internal/machine/api/pb/machine.pb.go +++ b/internal/machine/api/pb/machine.pb.go @@ -391,10 +391,11 @@ type JoinClusterRequest struct { Machine *MachineInfo `protobuf:"bytes,1,opt,name=machine,proto3" json:"machine,omitempty"` OtherMachines []*MachineInfo `protobuf:"bytes,3,rep,name=other_machines,json=otherMachines,proto3" json:"other_machines,omitempty"` - // Minimum store database version the new machine should sync to before starting cluster operations. - MinStoreDbVersion int64 `protobuf:"varint,4,opt,name=min_store_db_version,json=minStoreDbVersion,proto3" json:"min_store_db_version,omitempty"` // WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set. WireguardPort int32 `protobuf:"varint,5,opt,name=wireguard_port,json=wireguardPort,proto3" json:"wireguard_port,omitempty"` + // Cluster store version this machine must reach before participating. + // Per-actor vector (Corrosion actor UUID → max applied db_version). + MinStoreVersion map[string]int64 `protobuf:"bytes,6,rep,name=min_store_version,json=minStoreVersion,proto3" json:"min_store_version,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } func (x *JoinClusterRequest) Reset() { @@ -443,13 +444,6 @@ func (x *JoinClusterRequest) GetOtherMachines() []*MachineInfo { return nil } -func (x *JoinClusterRequest) GetMinStoreDbVersion() int64 { - if x != nil { - return x.MinStoreDbVersion - } - return 0 -} - func (x *JoinClusterRequest) GetWireguardPort() int32 { if x != nil { return x.WireguardPort @@ -457,6 +451,13 @@ func (x *JoinClusterRequest) GetWireguardPort() int32 { return 0 } +func (x *JoinClusterRequest) GetMinStoreVersion() map[string]int64 { + if x != nil { + return x.MinStoreVersion + } + return nil +} + type InspectMachineResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -512,10 +513,11 @@ type MachineDetails struct { Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` Machine *MachineInfo `protobuf:"bytes,2,opt,name=machine,proto3" json:"machine,omitempty"` - // Current Corrosion cr-sqlite database version (Lamport timestamp) of the cluster store. - StoreDbVersion int64 `protobuf:"varint,3,opt,name=store_db_version,json=storeDbVersion,proto3" json:"store_db_version,omitempty"` // Round-trip times to other machines in the cluster, keyed by peer machine ID. Rtts map[string]*RTTStats `protobuf:"bytes,4,rep,name=rtts,proto3" json:"rtts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Current cluster store version observed on this machine. + // Per-actor vector (Corrosion actor UUID → max applied db_version) read from Corrosion crsql_db_versions. + StoreVersion map[string]int64 `protobuf:"bytes,5,rep,name=store_version,json=storeVersion,proto3" json:"store_version,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } func (x *MachineDetails) Reset() { @@ -564,13 +566,6 @@ func (x *MachineDetails) GetMachine() *MachineInfo { return nil } -func (x *MachineDetails) GetStoreDbVersion() int64 { - if x != nil { - return x.StoreDbVersion - } - return 0 -} - func (x *MachineDetails) GetRtts() map[string]*RTTStats { if x != nil { return x.Rtts @@ -578,6 +573,13 @@ func (x *MachineDetails) GetRtts() map[string]*RTTStats { return nil } +func (x *MachineDetails) GetStoreVersion() map[string]int64 { + if x != nil { + return x.StoreVersion + } + return nil +} + type TokenResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1054,7 +1056,7 @@ type Service_Container struct { func (x *Service_Container) Reset() { *x = Service_Container{} if protoimpl.UnsafeEnabled { - mi := &file_internal_machine_api_pb_machine_proto_msgTypes[17] + mi := &file_internal_machine_api_pb_machine_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1067,7 +1069,7 @@ func (x *Service_Container) String() string { func (*Service_Container) ProtoMessage() {} func (x *Service_Container) ProtoReflect() protoreflect.Message { - mi := &file_internal_machine_api_pb_machine_proto_msgTypes[17] + mi := &file_internal_machine_api_pb_machine_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1158,7 +1160,7 @@ var file_internal_machine_api_pb_machine_proto_rawDesc = []byte{ 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0xd1, 0x01, 0x0a, 0x12, 0x4a, 0x6f, + 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0xbe, 0x02, 0x0a, 0x12, 0x4a, 0x6f, 0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, @@ -1166,33 +1168,46 @@ var file_internal_machine_api_pb_machine_proto_rawDesc = []byte{ 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x4d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x74, 0x6f, - 0x72, 0x65, 0x5f, 0x64, 0x62, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x11, 0x6d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x44, 0x62, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, - 0x61, 0x72, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, - 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x49, 0x0a, - 0x16, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, - 0x6e, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x08, - 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x22, 0x8c, 0x02, 0x0a, 0x0e, 0x4d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x29, 0x0a, 0x08, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, - 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, - 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, - 0x6e, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x64, 0x62, 0x5f, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x44, 0x62, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x04, - 0x72, 0x74, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x2e, - 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x72, 0x74, 0x74, 0x73, 0x1a, - 0x46, 0x0a, 0x09, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, - 0x61, 0x70, 0x69, 0x2e, 0x52, 0x54, 0x54, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, + 0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, + 0x72, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x77, + 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x58, 0x0a, 0x11, + 0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4a, 0x6f, + 0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x4d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x6d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x42, 0x0a, 0x14, 0x4d, 0x69, 0x6e, 0x53, 0x74, 0x6f, + 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x49, 0x0a, 0x16, 0x49, 0x6e, + 0x73, 0x70, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, + 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x08, 0x6d, 0x61, 0x63, + 0x68, 0x69, 0x6e, 0x65, 0x73, 0x22, 0xef, 0x02, 0x0a, 0x0e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, + 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x29, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, + 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, + 0x31, 0x0a, 0x04, 0x72, 0x74, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, + 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, + 0x6c, 0x73, 0x2e, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x72, 0x74, + 0x74, 0x73, 0x12, 0x4a, 0x0a, 0x0d, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x53, + 0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x46, + 0x0a, 0x09, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x52, 0x54, 0x54, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3f, 0x0a, 0x11, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25, 0x0a, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x0e, @@ -1311,7 +1326,7 @@ func file_internal_machine_api_pb_machine_proto_rawDescGZIP() []byte { return file_internal_machine_api_pb_machine_proto_rawDescData } -var file_internal_machine_api_pb_machine_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_internal_machine_api_pb_machine_proto_msgTypes = make([]protoimpl.MessageInfo, 20) var file_internal_machine_api_pb_machine_proto_goTypes = []any{ (*MachineInfo)(nil), // 0: api.MachineInfo (*NetworkConfig)(nil), // 1: api.NetworkConfig @@ -1329,66 +1344,70 @@ var file_internal_machine_api_pb_machine_proto_goTypes = []any{ (*InspectWireGuardNetworkResponse)(nil), // 13: api.InspectWireGuardNetworkResponse (*WireGuardPeer)(nil), // 14: api.WireGuardPeer (*RTTStats)(nil), // 15: api.RTTStats - nil, // 16: api.MachineDetails.RttsEntry - (*Service_Container)(nil), // 17: api.Service.Container - (*IP)(nil), // 18: api.IP - (*IPPrefix)(nil), // 19: api.IPPrefix - (*IPPort)(nil), // 20: api.IPPort - (*Metadata)(nil), // 21: api.Metadata - (*timestamppb.Timestamp)(nil), // 22: google.protobuf.Timestamp - (*durationpb.Duration)(nil), // 23: google.protobuf.Duration - (*emptypb.Empty)(nil), // 24: google.protobuf.Empty - (*LogsRequest)(nil), // 25: api.LogsRequest - (*LogEntry)(nil), // 26: api.LogEntry + nil, // 16: api.JoinClusterRequest.MinStoreVersionEntry + nil, // 17: api.MachineDetails.RttsEntry + nil, // 18: api.MachineDetails.StoreVersionEntry + (*Service_Container)(nil), // 19: api.Service.Container + (*IP)(nil), // 20: api.IP + (*IPPrefix)(nil), // 21: api.IPPrefix + (*IPPort)(nil), // 22: api.IPPort + (*Metadata)(nil), // 23: api.Metadata + (*timestamppb.Timestamp)(nil), // 24: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 25: google.protobuf.Duration + (*emptypb.Empty)(nil), // 26: google.protobuf.Empty + (*LogsRequest)(nil), // 27: api.LogsRequest + (*LogEntry)(nil), // 28: api.LogEntry } var file_internal_machine_api_pb_machine_proto_depIdxs = []int32{ 1, // 0: api.MachineInfo.network:type_name -> api.NetworkConfig - 18, // 1: api.MachineInfo.public_ip:type_name -> api.IP - 19, // 2: api.NetworkConfig.subnet:type_name -> api.IPPrefix - 18, // 3: api.NetworkConfig.management_ip:type_name -> api.IP - 20, // 4: api.NetworkConfig.endpoints:type_name -> api.IPPort - 19, // 5: api.InitClusterRequest.network:type_name -> api.IPPrefix - 18, // 6: api.InitClusterRequest.public_ip:type_name -> api.IP - 20, // 7: api.InitClusterRequest.wireguard_endpoints:type_name -> api.IPPort + 20, // 1: api.MachineInfo.public_ip:type_name -> api.IP + 21, // 2: api.NetworkConfig.subnet:type_name -> api.IPPrefix + 20, // 3: api.NetworkConfig.management_ip:type_name -> api.IP + 22, // 4: api.NetworkConfig.endpoints:type_name -> api.IPPort + 21, // 5: api.InitClusterRequest.network:type_name -> api.IPPrefix + 20, // 6: api.InitClusterRequest.public_ip:type_name -> api.IP + 22, // 7: api.InitClusterRequest.wireguard_endpoints:type_name -> api.IPPort 0, // 8: api.InitClusterResponse.machine:type_name -> api.MachineInfo 0, // 9: api.JoinClusterRequest.machine:type_name -> api.MachineInfo 0, // 10: api.JoinClusterRequest.other_machines:type_name -> api.MachineInfo - 7, // 11: api.InspectMachineResponse.machines:type_name -> api.MachineDetails - 21, // 12: api.MachineDetails.metadata:type_name -> api.Metadata - 0, // 13: api.MachineDetails.machine:type_name -> api.MachineInfo - 16, // 14: api.MachineDetails.rtts:type_name -> api.MachineDetails.RttsEntry - 17, // 15: api.Service.containers:type_name -> api.Service.Container - 10, // 16: api.InspectServiceResponse.service:type_name -> api.Service - 14, // 17: api.InspectWireGuardNetworkResponse.peers:type_name -> api.WireGuardPeer - 22, // 18: api.WireGuardPeer.last_handshake_time:type_name -> google.protobuf.Timestamp - 23, // 19: api.RTTStats.median:type_name -> google.protobuf.Duration - 23, // 20: api.RTTStats.std_dev:type_name -> google.protobuf.Duration - 15, // 21: api.MachineDetails.RttsEntry.value:type_name -> api.RTTStats - 24, // 22: api.Machine.CheckPrerequisites:input_type -> google.protobuf.Empty - 3, // 23: api.Machine.InitCluster:input_type -> api.InitClusterRequest - 5, // 24: api.Machine.JoinCluster:input_type -> api.JoinClusterRequest - 24, // 25: api.Machine.Token:input_type -> google.protobuf.Empty - 24, // 26: api.Machine.Inspect:input_type -> google.protobuf.Empty - 24, // 27: api.Machine.InspectMachine:input_type -> google.protobuf.Empty - 24, // 28: api.Machine.InspectWireGuardNetwork:input_type -> google.protobuf.Empty - 9, // 29: api.Machine.Reset:input_type -> api.ResetRequest - 11, // 30: api.Machine.InspectService:input_type -> api.InspectServiceRequest - 25, // 31: api.Machine.MachineLogs:input_type -> api.LogsRequest - 2, // 32: api.Machine.CheckPrerequisites:output_type -> api.CheckPrerequisitesResponse - 4, // 33: api.Machine.InitCluster:output_type -> api.InitClusterResponse - 24, // 34: api.Machine.JoinCluster:output_type -> google.protobuf.Empty - 8, // 35: api.Machine.Token:output_type -> api.TokenResponse - 0, // 36: api.Machine.Inspect:output_type -> api.MachineInfo - 6, // 37: api.Machine.InspectMachine:output_type -> api.InspectMachineResponse - 13, // 38: api.Machine.InspectWireGuardNetwork:output_type -> api.InspectWireGuardNetworkResponse - 24, // 39: api.Machine.Reset:output_type -> google.protobuf.Empty - 12, // 40: api.Machine.InspectService:output_type -> api.InspectServiceResponse - 26, // 41: api.Machine.MachineLogs:output_type -> api.LogEntry - 32, // [32:42] is the sub-list for method output_type - 22, // [22:32] is the sub-list for method input_type - 22, // [22:22] is the sub-list for extension type_name - 22, // [22:22] is the sub-list for extension extendee - 0, // [0:22] is the sub-list for field type_name + 16, // 11: api.JoinClusterRequest.min_store_version:type_name -> api.JoinClusterRequest.MinStoreVersionEntry + 7, // 12: api.InspectMachineResponse.machines:type_name -> api.MachineDetails + 23, // 13: api.MachineDetails.metadata:type_name -> api.Metadata + 0, // 14: api.MachineDetails.machine:type_name -> api.MachineInfo + 17, // 15: api.MachineDetails.rtts:type_name -> api.MachineDetails.RttsEntry + 18, // 16: api.MachineDetails.store_version:type_name -> api.MachineDetails.StoreVersionEntry + 19, // 17: api.Service.containers:type_name -> api.Service.Container + 10, // 18: api.InspectServiceResponse.service:type_name -> api.Service + 14, // 19: api.InspectWireGuardNetworkResponse.peers:type_name -> api.WireGuardPeer + 24, // 20: api.WireGuardPeer.last_handshake_time:type_name -> google.protobuf.Timestamp + 25, // 21: api.RTTStats.median:type_name -> google.protobuf.Duration + 25, // 22: api.RTTStats.std_dev:type_name -> google.protobuf.Duration + 15, // 23: api.MachineDetails.RttsEntry.value:type_name -> api.RTTStats + 26, // 24: api.Machine.CheckPrerequisites:input_type -> google.protobuf.Empty + 3, // 25: api.Machine.InitCluster:input_type -> api.InitClusterRequest + 5, // 26: api.Machine.JoinCluster:input_type -> api.JoinClusterRequest + 26, // 27: api.Machine.Token:input_type -> google.protobuf.Empty + 26, // 28: api.Machine.Inspect:input_type -> google.protobuf.Empty + 26, // 29: api.Machine.InspectMachine:input_type -> google.protobuf.Empty + 26, // 30: api.Machine.InspectWireGuardNetwork:input_type -> google.protobuf.Empty + 9, // 31: api.Machine.Reset:input_type -> api.ResetRequest + 11, // 32: api.Machine.InspectService:input_type -> api.InspectServiceRequest + 27, // 33: api.Machine.MachineLogs:input_type -> api.LogsRequest + 2, // 34: api.Machine.CheckPrerequisites:output_type -> api.CheckPrerequisitesResponse + 4, // 35: api.Machine.InitCluster:output_type -> api.InitClusterResponse + 26, // 36: api.Machine.JoinCluster:output_type -> google.protobuf.Empty + 8, // 37: api.Machine.Token:output_type -> api.TokenResponse + 0, // 38: api.Machine.Inspect:output_type -> api.MachineInfo + 6, // 39: api.Machine.InspectMachine:output_type -> api.InspectMachineResponse + 13, // 40: api.Machine.InspectWireGuardNetwork:output_type -> api.InspectWireGuardNetworkResponse + 26, // 41: api.Machine.Reset:output_type -> google.protobuf.Empty + 12, // 42: api.Machine.InspectService:output_type -> api.InspectServiceResponse + 28, // 43: api.Machine.MachineLogs:output_type -> api.LogEntry + 34, // [34:44] is the sub-list for method output_type + 24, // [24:34] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_internal_machine_api_pb_machine_proto_init() } @@ -1590,7 +1609,7 @@ func file_internal_machine_api_pb_machine_proto_init() { return nil } } - file_internal_machine_api_pb_machine_proto_msgTypes[17].Exporter = func(v any, i int) any { + file_internal_machine_api_pb_machine_proto_msgTypes[19].Exporter = func(v any, i int) any { switch v := v.(*Service_Container); i { case 0: return &v.state @@ -1613,7 +1632,7 @@ func file_internal_machine_api_pb_machine_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_internal_machine_api_pb_machine_proto_rawDesc, NumEnums: 0, - NumMessages: 18, + NumMessages: 20, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/machine/api/pb/machine.proto b/internal/machine/api/pb/machine.proto index 6e6db9cd..b848b029 100644 --- a/internal/machine/api/pb/machine.proto +++ b/internal/machine/api/pb/machine.proto @@ -73,10 +73,11 @@ message InitClusterResponse { message JoinClusterRequest { MachineInfo machine = 1; repeated MachineInfo other_machines = 3; - // Minimum store database version the new machine should sync to before starting cluster operations. - int64 min_store_db_version = 4; // WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set. int32 wireguard_port = 5; + // Cluster store version this machine must reach before participating. + // Per-actor vector (Corrosion actor UUID → max applied db_version). + map min_store_version = 6; } message InspectMachineResponse { @@ -87,10 +88,11 @@ message InspectMachineResponse { message MachineDetails { Metadata metadata = 1; MachineInfo machine = 2; - // Current Corrosion cr-sqlite database version (Lamport timestamp) of the cluster store. - int64 store_db_version = 3; // Round-trip times to other machines in the cluster, keyed by peer machine ID. map rtts = 4; + // Current cluster store version observed on this machine. + // Per-actor vector (Corrosion actor UUID → max applied db_version) read from Corrosion crsql_db_versions. + map store_version = 5; } message TokenResponse { diff --git a/internal/machine/cluster.go b/internal/machine/cluster.go index 592c4602..13093209 100644 --- a/internal/machine/cluster.go +++ b/internal/machine/cluster.go @@ -96,7 +96,8 @@ func newClusterController( func (cc *clusterController) Run(ctx context.Context) error { defer close(cc.stopped) - if err := firewall.ConfigureIptablesChains(network.MachineIP(cc.state.Network.Subnet), cc.state.Network.EffectiveWireGuardPort()); err != nil { + if err := firewall.ConfigureIptablesChains(network.MachineIP(cc.state.Network.Subnet), + cc.state.Network.EffectiveWireGuardPort()); err != nil { return fmt.Errorf("configure iptables chains: %w", err) } @@ -159,7 +160,9 @@ func (cc *clusterController) Run(ctx context.Context) error { // Wait for the store database to sync to the minimum version before starting store-dependent components. // This prevents issues with using partially replicated data when the machine just joined the cluster, // e.g., an empty machine list causing WireGuard peer misconfiguration. - cc.waitStoreSync(ctx) + if err = cc.waitStoreSync(ctx); err != nil { + return fmt.Errorf("wait initial cluster store sync: %w", err) + } // Check if waitStoreSync exited because the context was cancelled. Return early in that case. if ctx.Err() != nil { @@ -341,101 +344,114 @@ func (cc *clusterController) handleEndpointChanges(ctx context.Context) { } } -// waitStoreSync waits for the store database to sync to the minimum required DB version if set in the machine state. -// Blocks until synced or context is cancelled. -func (cc *clusterController) waitStoreSync(ctx context.Context) { - minVersion := cc.state.MinStoreDBVersion - if minVersion == 0 { - return +// waitStoreSync blocks until the local store version >= state.MinStoreVersion and any known gaps are synced. +// No-op when MinStoreVersion is empty. Clears state.MinStoreVersion when reached. +func (cc *clusterController) waitStoreSync(ctx context.Context) error { + target := cc.state.MinStoreVersion + if len(target) == 0 { + return nil } - slog.Info("Waiting for the initial cluster store sync.", "min_version", minVersion) + slog.Info("Waiting for the initial cluster store sync.", "actors", len(target)) ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() + // Periodic warning to surface stuck NAT/connectivity issues without aborting. + warnInterval := 5 * time.Minute + warnTimer := time.NewTimer(warnInterval) + defer warnTimer.Stop() var ( - lastVersion int64 - lastLogTime time.Time + lastLagging int lastErrLogTime time.Time ) for { select { case <-ctx.Done(): - return + return nil + case <-warnTimer.C: + local, err := cc.store.Version(ctx) + if err == nil { + slog.Error("Cluster store sync still pending. Check connectivity to peers.", + "lagging_actors", laggingActors(local, target)) + } else { + slog.Error("Cluster store sync still pending. Check connectivity to peers.", "err", err) + } + warnTimer.Reset(warnInterval) case <-ticker.C: - version, err := cc.store.DBVersion(ctx) + local, err := cc.store.Version(ctx) if err != nil { - // Log errors at most once every 5 seconds. + // Throttle error logs to once every 5 seconds. if time.Since(lastErrLogTime) >= 5*time.Second { - slog.Error("Failed to get the cluster store DB version, retrying.", "err", err) + slog.Error("Failed to get the cluster store version, retrying.", "err", err) lastErrLogTime = time.Now() } continue } - if version >= minVersion { - // Clear MinStoreDBVersion so next restart doesn't wait for sync. + lagging := laggingActors(local, target) + if len(lagging) == 0 { + // Per-actor max doesn't imply contiguous apply: corrosion can buffer X:N before + // X:N-1 arrives and track the gap separately. Wait for any remaining gaps to be synced. + if err := cc.waitKnownMissingChanges(ctx); err != nil { + return fmt.Errorf("wait for known missing changes: %w", err) + } + // If the context was cancelled mid-gap-fill, don't persist a "synced" state. + if ctx.Err() != nil { + return nil + } + + // Clear MinStoreVersion so next restart doesn't wait for sync. cc.state.mu.Lock() - cc.state.MinStoreDBVersion = 0 - if err := cc.state.Save(); err != nil { - slog.Error("Failed to save machine state after the initial cluster store sync.", "err", err) - } + cc.state.MinStoreVersion = nil + err = cc.state.Save() cc.state.mu.Unlock() - - // Wait for all known missing changes to be synced before returning. - // TODO: reevaluate if this is necessary after migrating to the latest Corrosion version: - // https://github.com/psviderski/uncloud/issues/172 - // This works on the best effort basis as the missing changes may not be yet known when we start - // checking it after reaching the minimum version. - // Reaching the minimum version doesn't guarantee that the store is actually synced to - // the state we observed on the source node. db_version is a machine-local Lamport clock. - // When changes are received from a remote machine, the local db_version is set to - // max(local_db_version, incoming_db_version) + 1 for each applied transaction. This means - // the new machine's db_version can jump well past the source machine's db_version on the very - // first batch of replicated changes, without having received all changes from all machines - // in the cluster. - cc.waitKnownMissingChanges(ctx) - - if ver, verErr := cc.store.DBVersion(ctx); verErr == nil { - version = ver + if err != nil { + return fmt.Errorf("save machine state after the initial cluster store sync: %w", err) } - slog.Info("Cluster store completed the initial sync.", "version", version, "min_version", minVersion) - return + slog.Info("Cluster store completed the initial sync.", "actors", len(target)) + return nil } - // Log progress only once a second. - if version != lastVersion && time.Since(lastLogTime) >= 1*time.Second { - slog.Info("Syncing cluster store.", "version", version, "min_version", minVersion) - lastLogTime = time.Now() - lastVersion = version + if len(lagging) != lastLagging { + slog.Info("Syncing cluster store.", "lagging_actors", lagging) + lastLagging = len(lagging) } } } } +// laggingActors returns target actors whose local version is below the required value, as [have, need]. +func laggingActors(local, target map[string]int64) map[string][2]int64 { + lagging := make(map[string][2]int64) + for actor, need := range target { + if have := local[actor]; have < need { + lagging[actor] = [2]int64{have, need} + } + } + return lagging +} + // waitKnownMissingChanges polls the store until all known missing changes have been synced. -func (cc *clusterController) waitKnownMissingChanges(ctx context.Context) { +func (cc *clusterController) waitKnownMissingChanges(ctx context.Context) error { ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): - return + return nil case <-ticker.C: changes, err := cc.store.KnownMissingChanges(ctx) if err != nil { - slog.Error("Failed to get known missing changes from the cluster store, skipping check.", - "err", err) - return + return fmt.Errorf("query known missing changes from cluster store: %w", err) } if len(changes) == 0 { slog.Debug("All known missing changes have been synced to the cluster store.") - return + return nil } slog.Debug("Waiting for known missing changes to be synced to the cluster store.", "remaining", diff --git a/internal/machine/machine.go b/internal/machine/machine.go index 442714ff..f8a74504 100644 --- a/internal/machine/machine.go +++ b/internal/machine/machine.go @@ -841,7 +841,7 @@ func (m *Machine) JoinCluster(_ context.Context, req *pb.JoinClusterRequest) (*e PrivateKey: m.state.Network.PrivateKey, PublicKey: m.state.Network.PublicKey, } - m.state.MinStoreDBVersion = req.MinStoreDbVersion + m.state.MinStoreVersion = req.MinStoreVersion // Build a peers config from other cluster machines. m.state.Network.Peers = make([]network.PeerConfig, 0, len(req.OtherMachines)) @@ -928,9 +928,9 @@ func (m *Machine) Inspect(_ context.Context, _ *emptypb.Empty) (*pb.MachineInfo, } func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.InspectMachineResponse, error) { - dbVersion, err := m.store.DBVersion(ctx) + storeVersion, err := m.store.Version(ctx) if err != nil { - return nil, status.Errorf(codes.Internal, "get database version of the cluster store: %v", err) + return nil, status.Errorf(codes.Internal, "get cluster store version: %v", err) } var rtts map[string]*pb.RTTStats @@ -954,8 +954,8 @@ func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.Ins PublicKey: m.state.Network.PublicKey, }, }, - StoreDbVersion: dbVersion, - Rtts: rtts, + StoreVersion: storeVersion, + Rtts: rtts, }, }, }, nil diff --git a/internal/machine/state.go b/internal/machine/state.go index e05159f7..25892834 100644 --- a/internal/machine/state.go +++ b/internal/machine/state.go @@ -24,9 +24,10 @@ type State struct { Name string // Network specifies the network configuration for this machine. Network *network.Config - // MinStoreDBVersion is the latest database version of one of the existing cluster machines at the time this machine - // joined the cluster. The machine should sync to at least this version before starting any cluster operations. - MinStoreDBVersion int64 `json:",omitempty"` + // MinStoreVersion is the cluster store version this machine must reach before participating. + // Per-actor vector (Corrosion actor UUID → max applied db_version) captured from an existing + // member at join time. Cleared once reached. + MinStoreVersion map[string]int64 `json:",omitempty"` // path is the file path config is read from and saved to. path string diff --git a/internal/machine/store/store.go b/internal/machine/store/store.go index 78843b44..b10510bf 100644 --- a/internal/machine/store/store.go +++ b/internal/machine/store/store.go @@ -8,6 +8,7 @@ import ( "fmt" "log/slog" + "github.com/google/uuid" "github.com/psviderski/uncloud/internal/corrosion" "github.com/psviderski/uncloud/internal/machine/api/pb" "google.golang.org/protobuf/encoding/protojson" @@ -57,23 +58,31 @@ func (s *Store) Delete(ctx context.Context, key string) error { return err } -// DBVersion returns the current cr-sqlite database version (Lamport timestamp). -func (s *Store) DBVersion(ctx context.Context) (int64, error) { - rows, err := s.corro.QueryContext(ctx, "SELECT crsql_db_version()") +// Version returns the cluster store's per-actor version vector: +// Corrosion actor ID (UUID string) → max received db_version for that actor. +func (s *Store) Version(ctx context.Context) (map[string]int64, error) { + rows, err := s.corro.QueryContext(ctx, "SELECT site_id, db_version FROM crsql_db_versions") if err != nil { - return 0, fmt.Errorf("query crsql_db_version(): %w", err) + return nil, fmt.Errorf("query crsql_db_versions: %w", err) } defer rows.Close() - if !rows.Next() { - return 0, fmt.Errorf("no result from crsql_db_version()") + versions := make(map[string]int64) + for rows.Next() { + var ( + siteID []byte + version int64 + ) + if err = rows.Scan(&siteID, &version); err != nil { + return nil, fmt.Errorf("scan actor version: %w", err) + } + actor, err := uuid.FromBytes(siteID) + if err != nil { + return nil, fmt.Errorf("parse site_id as UUID: %w", err) + } + versions[actor.String()] = version } - - var version int64 - if err = rows.Scan(&version); err != nil { - return 0, fmt.Errorf("scan db version: %w", err) - } - return version, nil + return versions, nil } type MissingChange struct { diff --git a/internal/ucind/cluster.go b/internal/ucind/cluster.go index c4b9d224..c805d2a2 100644 --- a/internal/ucind/cluster.go +++ b/internal/ucind/cluster.go @@ -192,9 +192,9 @@ func (p *Provisioner) initCluster(ctx context.Context, machines []Machine) error // Configure the machine to join the cluster. joinReq := &pb.JoinClusterRequest{ - Machine: addResp.Machine, - OtherMachines: []*pb.MachineInfo{initResp.Machine}, - MinStoreDbVersion: inspectResp.Machines[0].StoreDbVersion, + Machine: addResp.Machine, + OtherMachines: []*pb.MachineInfo{initResp.Machine}, + MinStoreVersion: inspectResp.Machines[0].StoreVersion, } if _, err = cli.JoinCluster(ctx, joinReq); err != nil { return fmt.Errorf("join cluster: %w", err)