refactor: migrate to latest Corrosion version 2026.5.14 for ucind (compare versions vector for store sync)

This commit is contained in:
Pasha Sviderski
2026-05-25 14:09:51 +10:00
parent 0dd4bc9d36
commit 9236aa341a
8 changed files with 237 additions and 190 deletions
+7 -7
View File
@@ -434,8 +434,8 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
return nil, nil, fmt.Errorf("add machine to cluster (context '%s'): %w", contextName, err) return nil, nil, fmt.Errorf("add machine to cluster (context '%s'): %w", contextName, err)
} }
// Get the current store DB version from the cluster to pass to the join request. // Snapshot the cluster store version so the new machine can catch up before participating.
var storeDBVersion int64 var storeVersion map[string]int64
inspectResp, err := c.MachineClient.InspectMachine(ctx, &emptypb.Empty{}) inspectResp, err := c.MachineClient.InspectMachine(ctx, &emptypb.Empty{})
if err != nil { if err != nil {
// TODO(lhf): remove Unimplemented check when v0.17.0 is released. // TODO(lhf): remove Unimplemented check when v0.17.0 is released.
@@ -443,7 +443,7 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
return nil, nil, fmt.Errorf("inspect current cluster machine: %w", err) return nil, nil, fmt.Errorf("inspect current cluster machine: %w", err)
} }
} else { } else {
storeDBVersion = inspectResp.Machines[0].StoreDbVersion storeVersion = inspectResp.Machines[0].StoreVersion
} }
// Get the most up-to-date list of other machines in the cluster to include them in the join request. // Get the most up-to-date list of other machines in the cluster to include them in the join request.
@@ -460,10 +460,10 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
// Configure the remote machine to join the cluster. // Configure the remote machine to join the cluster.
joinReq := &pb.JoinClusterRequest{ joinReq := &pb.JoinClusterRequest{
Machine: addResp.Machine, Machine: addResp.Machine,
OtherMachines: otherMachines, OtherMachines: otherMachines,
MinStoreDbVersion: storeDBVersion, MinStoreVersion: storeVersion,
WireguardPort: int32(opts.WireguardPort), WireguardPort: int32(opts.WireguardPort),
} }
if _, err = machineClient.JoinCluster(ctx, joinReq); err != nil { if _, err = machineClient.JoinCluster(ctx, joinReq); err != nil {
return nil, nil, fmt.Errorf("join cluster: %w", err) return nil, nil, fmt.Errorf("join cluster: %w", err)
+124 -105
View File
@@ -391,10 +391,11 @@ type JoinClusterRequest struct {
Machine *MachineInfo `protobuf:"bytes,1,opt,name=machine,proto3" json:"machine,omitempty"` Machine *MachineInfo `protobuf:"bytes,1,opt,name=machine,proto3" json:"machine,omitempty"`
OtherMachines []*MachineInfo `protobuf:"bytes,3,rep,name=other_machines,json=otherMachines,proto3" json:"other_machines,omitempty"` OtherMachines []*MachineInfo `protobuf:"bytes,3,rep,name=other_machines,json=otherMachines,proto3" json:"other_machines,omitempty"`
// Minimum store database version the new machine should sync to before starting cluster operations.
MinStoreDbVersion int64 `protobuf:"varint,4,opt,name=min_store_db_version,json=minStoreDbVersion,proto3" json:"min_store_db_version,omitempty"`
// WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set. // WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set.
WireguardPort int32 `protobuf:"varint,5,opt,name=wireguard_port,json=wireguardPort,proto3" json:"wireguard_port,omitempty"` WireguardPort int32 `protobuf:"varint,5,opt,name=wireguard_port,json=wireguardPort,proto3" json:"wireguard_port,omitempty"`
// Cluster store version this machine must reach before participating.
// Per-actor vector (Corrosion actor UUID → max applied db_version).
MinStoreVersion map[string]int64 `protobuf:"bytes,6,rep,name=min_store_version,json=minStoreVersion,proto3" json:"min_store_version,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
} }
func (x *JoinClusterRequest) Reset() { func (x *JoinClusterRequest) Reset() {
@@ -443,13 +444,6 @@ func (x *JoinClusterRequest) GetOtherMachines() []*MachineInfo {
return nil return nil
} }
func (x *JoinClusterRequest) GetMinStoreDbVersion() int64 {
if x != nil {
return x.MinStoreDbVersion
}
return 0
}
func (x *JoinClusterRequest) GetWireguardPort() int32 { func (x *JoinClusterRequest) GetWireguardPort() int32 {
if x != nil { if x != nil {
return x.WireguardPort return x.WireguardPort
@@ -457,6 +451,13 @@ func (x *JoinClusterRequest) GetWireguardPort() int32 {
return 0 return 0
} }
func (x *JoinClusterRequest) GetMinStoreVersion() map[string]int64 {
if x != nil {
return x.MinStoreVersion
}
return nil
}
type InspectMachineResponse struct { type InspectMachineResponse struct {
state protoimpl.MessageState state protoimpl.MessageState
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
@@ -512,10 +513,11 @@ type MachineDetails struct {
Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
Machine *MachineInfo `protobuf:"bytes,2,opt,name=machine,proto3" json:"machine,omitempty"` Machine *MachineInfo `protobuf:"bytes,2,opt,name=machine,proto3" json:"machine,omitempty"`
// Current Corrosion cr-sqlite database version (Lamport timestamp) of the cluster store.
StoreDbVersion int64 `protobuf:"varint,3,opt,name=store_db_version,json=storeDbVersion,proto3" json:"store_db_version,omitempty"`
// Round-trip times to other machines in the cluster, keyed by peer machine ID. // Round-trip times to other machines in the cluster, keyed by peer machine ID.
Rtts map[string]*RTTStats `protobuf:"bytes,4,rep,name=rtts,proto3" json:"rtts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` Rtts map[string]*RTTStats `protobuf:"bytes,4,rep,name=rtts,proto3" json:"rtts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// Current cluster store version observed on this machine.
// Per-actor vector (Corrosion actor UUID → max applied db_version) read from Corrosion crsql_db_versions.
StoreVersion map[string]int64 `protobuf:"bytes,5,rep,name=store_version,json=storeVersion,proto3" json:"store_version,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
} }
func (x *MachineDetails) Reset() { func (x *MachineDetails) Reset() {
@@ -564,13 +566,6 @@ func (x *MachineDetails) GetMachine() *MachineInfo {
return nil return nil
} }
func (x *MachineDetails) GetStoreDbVersion() int64 {
if x != nil {
return x.StoreDbVersion
}
return 0
}
func (x *MachineDetails) GetRtts() map[string]*RTTStats { func (x *MachineDetails) GetRtts() map[string]*RTTStats {
if x != nil { if x != nil {
return x.Rtts return x.Rtts
@@ -578,6 +573,13 @@ func (x *MachineDetails) GetRtts() map[string]*RTTStats {
return nil return nil
} }
func (x *MachineDetails) GetStoreVersion() map[string]int64 {
if x != nil {
return x.StoreVersion
}
return nil
}
type TokenResponse struct { type TokenResponse struct {
state protoimpl.MessageState state protoimpl.MessageState
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
@@ -1054,7 +1056,7 @@ type Service_Container struct {
func (x *Service_Container) Reset() { func (x *Service_Container) Reset() {
*x = Service_Container{} *x = Service_Container{}
if protoimpl.UnsafeEnabled { if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_machine_proto_msgTypes[17] mi := &file_internal_machine_api_pb_machine_proto_msgTypes[19]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
} }
@@ -1067,7 +1069,7 @@ func (x *Service_Container) String() string {
func (*Service_Container) ProtoMessage() {} func (*Service_Container) ProtoMessage() {}
func (x *Service_Container) ProtoReflect() protoreflect.Message { func (x *Service_Container) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_machine_proto_msgTypes[17] mi := &file_internal_machine_api_pb_machine_proto_msgTypes[19]
if protoimpl.UnsafeEnabled && x != nil { if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
@@ -1158,7 +1160,7 @@ var file_internal_machine_api_pb_machine_proto_rawDesc = []byte{
0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a,
0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10,
0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f,
0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0xd1, 0x01, 0x0a, 0x12, 0x4a, 0x6f, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0xbe, 0x02, 0x0a, 0x12, 0x4a, 0x6f,
0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49,
@@ -1166,33 +1168,46 @@ var file_internal_machine_api_pb_machine_proto_rawDesc = []byte{
0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x03, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x03,
0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69,
0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x4d, 0x61, 0x63, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x4d, 0x61, 0x63,
0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x74, 0x6f, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61,
0x72, 0x65, 0x5f, 0x64, 0x62, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x72, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x77,
0x01, 0x28, 0x03, 0x52, 0x11, 0x6d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x44, 0x62, 0x56, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x58, 0x0a, 0x11,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
0x61, 0x72, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x6e, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4a, 0x6f,
0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x49, 0x0a, 0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x16, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x2e, 0x4d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x6d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56,
0x6e, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x42, 0x0a, 0x14, 0x4d, 0x69, 0x6e, 0x53, 0x74, 0x6f,
0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x08, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x22, 0x8c, 0x02, 0x0a, 0x0e, 0x4d, 0x61, 0x63, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x29, 0x0a, 0x08, 0x6d, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52,
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x49, 0x0a, 0x16, 0x49, 0x6e,
0x61, 0x70, 0x69, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x73, 0x70, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70,
0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73,
0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63,
0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x08, 0x6d, 0x61, 0x63,
0x6e, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x64, 0x62, 0x5f, 0x76, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x22, 0xef, 0x02, 0x0a, 0x0e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x73, 0x74, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x29, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61,
0x6f, 0x72, 0x65, 0x44, 0x62, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61, 0x70, 0x69,
0x72, 0x74, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64,
0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x61, 0x74, 0x61, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x02,
0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x72, 0x74, 0x74, 0x73, 0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69,
0x46, 0x0a, 0x09, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12,
0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x31, 0x0a, 0x04, 0x72, 0x74, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e,
0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69,
0x61, 0x70, 0x69, 0x2e, 0x52, 0x54, 0x54, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x73, 0x2e, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x72, 0x74,
0x74, 0x73, 0x12, 0x4a, 0x0a, 0x0d, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73,
0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x2e,
0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x53,
0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79,
0x52, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x46,
0x0a, 0x09, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x0a,
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61,
0x70, 0x69, 0x2e, 0x52, 0x54, 0x54, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c,
0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3f, 0x0a, 0x11, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a,
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61,
0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25, 0x0a, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25, 0x0a, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65,
0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x0e, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x0e,
@@ -1311,7 +1326,7 @@ func file_internal_machine_api_pb_machine_proto_rawDescGZIP() []byte {
return file_internal_machine_api_pb_machine_proto_rawDescData return file_internal_machine_api_pb_machine_proto_rawDescData
} }
var file_internal_machine_api_pb_machine_proto_msgTypes = make([]protoimpl.MessageInfo, 18) var file_internal_machine_api_pb_machine_proto_msgTypes = make([]protoimpl.MessageInfo, 20)
var file_internal_machine_api_pb_machine_proto_goTypes = []any{ var file_internal_machine_api_pb_machine_proto_goTypes = []any{
(*MachineInfo)(nil), // 0: api.MachineInfo (*MachineInfo)(nil), // 0: api.MachineInfo
(*NetworkConfig)(nil), // 1: api.NetworkConfig (*NetworkConfig)(nil), // 1: api.NetworkConfig
@@ -1329,66 +1344,70 @@ var file_internal_machine_api_pb_machine_proto_goTypes = []any{
(*InspectWireGuardNetworkResponse)(nil), // 13: api.InspectWireGuardNetworkResponse (*InspectWireGuardNetworkResponse)(nil), // 13: api.InspectWireGuardNetworkResponse
(*WireGuardPeer)(nil), // 14: api.WireGuardPeer (*WireGuardPeer)(nil), // 14: api.WireGuardPeer
(*RTTStats)(nil), // 15: api.RTTStats (*RTTStats)(nil), // 15: api.RTTStats
nil, // 16: api.MachineDetails.RttsEntry nil, // 16: api.JoinClusterRequest.MinStoreVersionEntry
(*Service_Container)(nil), // 17: api.Service.Container nil, // 17: api.MachineDetails.RttsEntry
(*IP)(nil), // 18: api.IP nil, // 18: api.MachineDetails.StoreVersionEntry
(*IPPrefix)(nil), // 19: api.IPPrefix (*Service_Container)(nil), // 19: api.Service.Container
(*IPPort)(nil), // 20: api.IPPort (*IP)(nil), // 20: api.IP
(*Metadata)(nil), // 21: api.Metadata (*IPPrefix)(nil), // 21: api.IPPrefix
(*timestamppb.Timestamp)(nil), // 22: google.protobuf.Timestamp (*IPPort)(nil), // 22: api.IPPort
(*durationpb.Duration)(nil), // 23: google.protobuf.Duration (*Metadata)(nil), // 23: api.Metadata
(*emptypb.Empty)(nil), // 24: google.protobuf.Empty (*timestamppb.Timestamp)(nil), // 24: google.protobuf.Timestamp
(*LogsRequest)(nil), // 25: api.LogsRequest (*durationpb.Duration)(nil), // 25: google.protobuf.Duration
(*LogEntry)(nil), // 26: api.LogEntry (*emptypb.Empty)(nil), // 26: google.protobuf.Empty
(*LogsRequest)(nil), // 27: api.LogsRequest
(*LogEntry)(nil), // 28: api.LogEntry
} }
var file_internal_machine_api_pb_machine_proto_depIdxs = []int32{ var file_internal_machine_api_pb_machine_proto_depIdxs = []int32{
1, // 0: api.MachineInfo.network:type_name -> api.NetworkConfig 1, // 0: api.MachineInfo.network:type_name -> api.NetworkConfig
18, // 1: api.MachineInfo.public_ip:type_name -> api.IP 20, // 1: api.MachineInfo.public_ip:type_name -> api.IP
19, // 2: api.NetworkConfig.subnet:type_name -> api.IPPrefix 21, // 2: api.NetworkConfig.subnet:type_name -> api.IPPrefix
18, // 3: api.NetworkConfig.management_ip:type_name -> api.IP 20, // 3: api.NetworkConfig.management_ip:type_name -> api.IP
20, // 4: api.NetworkConfig.endpoints:type_name -> api.IPPort 22, // 4: api.NetworkConfig.endpoints:type_name -> api.IPPort
19, // 5: api.InitClusterRequest.network:type_name -> api.IPPrefix 21, // 5: api.InitClusterRequest.network:type_name -> api.IPPrefix
18, // 6: api.InitClusterRequest.public_ip:type_name -> api.IP 20, // 6: api.InitClusterRequest.public_ip:type_name -> api.IP
20, // 7: api.InitClusterRequest.wireguard_endpoints:type_name -> api.IPPort 22, // 7: api.InitClusterRequest.wireguard_endpoints:type_name -> api.IPPort
0, // 8: api.InitClusterResponse.machine:type_name -> api.MachineInfo 0, // 8: api.InitClusterResponse.machine:type_name -> api.MachineInfo
0, // 9: api.JoinClusterRequest.machine:type_name -> api.MachineInfo 0, // 9: api.JoinClusterRequest.machine:type_name -> api.MachineInfo
0, // 10: api.JoinClusterRequest.other_machines:type_name -> api.MachineInfo 0, // 10: api.JoinClusterRequest.other_machines:type_name -> api.MachineInfo
7, // 11: api.InspectMachineResponse.machines:type_name -> api.MachineDetails 16, // 11: api.JoinClusterRequest.min_store_version:type_name -> api.JoinClusterRequest.MinStoreVersionEntry
21, // 12: api.MachineDetails.metadata:type_name -> api.Metadata 7, // 12: api.InspectMachineResponse.machines:type_name -> api.MachineDetails
0, // 13: api.MachineDetails.machine:type_name -> api.MachineInfo 23, // 13: api.MachineDetails.metadata:type_name -> api.Metadata
16, // 14: api.MachineDetails.rtts:type_name -> api.MachineDetails.RttsEntry 0, // 14: api.MachineDetails.machine:type_name -> api.MachineInfo
17, // 15: api.Service.containers:type_name -> api.Service.Container 17, // 15: api.MachineDetails.rtts:type_name -> api.MachineDetails.RttsEntry
10, // 16: api.InspectServiceResponse.service:type_name -> api.Service 18, // 16: api.MachineDetails.store_version:type_name -> api.MachineDetails.StoreVersionEntry
14, // 17: api.InspectWireGuardNetworkResponse.peers:type_name -> api.WireGuardPeer 19, // 17: api.Service.containers:type_name -> api.Service.Container
22, // 18: api.WireGuardPeer.last_handshake_time:type_name -> google.protobuf.Timestamp 10, // 18: api.InspectServiceResponse.service:type_name -> api.Service
23, // 19: api.RTTStats.median:type_name -> google.protobuf.Duration 14, // 19: api.InspectWireGuardNetworkResponse.peers:type_name -> api.WireGuardPeer
23, // 20: api.RTTStats.std_dev:type_name -> google.protobuf.Duration 24, // 20: api.WireGuardPeer.last_handshake_time:type_name -> google.protobuf.Timestamp
15, // 21: api.MachineDetails.RttsEntry.value:type_name -> api.RTTStats 25, // 21: api.RTTStats.median:type_name -> google.protobuf.Duration
24, // 22: api.Machine.CheckPrerequisites:input_type -> google.protobuf.Empty 25, // 22: api.RTTStats.std_dev:type_name -> google.protobuf.Duration
3, // 23: api.Machine.InitCluster:input_type -> api.InitClusterRequest 15, // 23: api.MachineDetails.RttsEntry.value:type_name -> api.RTTStats
5, // 24: api.Machine.JoinCluster:input_type -> api.JoinClusterRequest 26, // 24: api.Machine.CheckPrerequisites:input_type -> google.protobuf.Empty
24, // 25: api.Machine.Token:input_type -> google.protobuf.Empty 3, // 25: api.Machine.InitCluster:input_type -> api.InitClusterRequest
24, // 26: api.Machine.Inspect:input_type -> google.protobuf.Empty 5, // 26: api.Machine.JoinCluster:input_type -> api.JoinClusterRequest
24, // 27: api.Machine.InspectMachine:input_type -> google.protobuf.Empty 26, // 27: api.Machine.Token:input_type -> google.protobuf.Empty
24, // 28: api.Machine.InspectWireGuardNetwork:input_type -> google.protobuf.Empty 26, // 28: api.Machine.Inspect:input_type -> google.protobuf.Empty
9, // 29: api.Machine.Reset:input_type -> api.ResetRequest 26, // 29: api.Machine.InspectMachine:input_type -> google.protobuf.Empty
11, // 30: api.Machine.InspectService:input_type -> api.InspectServiceRequest 26, // 30: api.Machine.InspectWireGuardNetwork:input_type -> google.protobuf.Empty
25, // 31: api.Machine.MachineLogs:input_type -> api.LogsRequest 9, // 31: api.Machine.Reset:input_type -> api.ResetRequest
2, // 32: api.Machine.CheckPrerequisites:output_type -> api.CheckPrerequisitesResponse 11, // 32: api.Machine.InspectService:input_type -> api.InspectServiceRequest
4, // 33: api.Machine.InitCluster:output_type -> api.InitClusterResponse 27, // 33: api.Machine.MachineLogs:input_type -> api.LogsRequest
24, // 34: api.Machine.JoinCluster:output_type -> google.protobuf.Empty 2, // 34: api.Machine.CheckPrerequisites:output_type -> api.CheckPrerequisitesResponse
8, // 35: api.Machine.Token:output_type -> api.TokenResponse 4, // 35: api.Machine.InitCluster:output_type -> api.InitClusterResponse
0, // 36: api.Machine.Inspect:output_type -> api.MachineInfo 26, // 36: api.Machine.JoinCluster:output_type -> google.protobuf.Empty
6, // 37: api.Machine.InspectMachine:output_type -> api.InspectMachineResponse 8, // 37: api.Machine.Token:output_type -> api.TokenResponse
13, // 38: api.Machine.InspectWireGuardNetwork:output_type -> api.InspectWireGuardNetworkResponse 0, // 38: api.Machine.Inspect:output_type -> api.MachineInfo
24, // 39: api.Machine.Reset:output_type -> google.protobuf.Empty 6, // 39: api.Machine.InspectMachine:output_type -> api.InspectMachineResponse
12, // 40: api.Machine.InspectService:output_type -> api.InspectServiceResponse 13, // 40: api.Machine.InspectWireGuardNetwork:output_type -> api.InspectWireGuardNetworkResponse
26, // 41: api.Machine.MachineLogs:output_type -> api.LogEntry 26, // 41: api.Machine.Reset:output_type -> google.protobuf.Empty
32, // [32:42] is the sub-list for method output_type 12, // 42: api.Machine.InspectService:output_type -> api.InspectServiceResponse
22, // [22:32] is the sub-list for method input_type 28, // 43: api.Machine.MachineLogs:output_type -> api.LogEntry
22, // [22:22] is the sub-list for extension type_name 34, // [34:44] is the sub-list for method output_type
22, // [22:22] is the sub-list for extension extendee 24, // [24:34] is the sub-list for method input_type
0, // [0:22] is the sub-list for field type_name 24, // [24:24] is the sub-list for extension type_name
24, // [24:24] is the sub-list for extension extendee
0, // [0:24] is the sub-list for field type_name
} }
func init() { file_internal_machine_api_pb_machine_proto_init() } func init() { file_internal_machine_api_pb_machine_proto_init() }
@@ -1590,7 +1609,7 @@ func file_internal_machine_api_pb_machine_proto_init() {
return nil return nil
} }
} }
file_internal_machine_api_pb_machine_proto_msgTypes[17].Exporter = func(v any, i int) any { file_internal_machine_api_pb_machine_proto_msgTypes[19].Exporter = func(v any, i int) any {
switch v := v.(*Service_Container); i { switch v := v.(*Service_Container); i {
case 0: case 0:
return &v.state return &v.state
@@ -1613,7 +1632,7 @@ func file_internal_machine_api_pb_machine_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(), GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_internal_machine_api_pb_machine_proto_rawDesc, RawDescriptor: file_internal_machine_api_pb_machine_proto_rawDesc,
NumEnums: 0, NumEnums: 0,
NumMessages: 18, NumMessages: 20,
NumExtensions: 0, NumExtensions: 0,
NumServices: 1, NumServices: 1,
}, },
+6 -4
View File
@@ -73,10 +73,11 @@ message InitClusterResponse {
message JoinClusterRequest { message JoinClusterRequest {
MachineInfo machine = 1; MachineInfo machine = 1;
repeated MachineInfo other_machines = 3; repeated MachineInfo other_machines = 3;
// Minimum store database version the new machine should sync to before starting cluster operations.
int64 min_store_db_version = 4;
// WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set. // WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set.
int32 wireguard_port = 5; int32 wireguard_port = 5;
// Cluster store version this machine must reach before participating.
// Per-actor vector (Corrosion actor UUID → max applied db_version).
map<string, int64> min_store_version = 6;
} }
message InspectMachineResponse { message InspectMachineResponse {
@@ -87,10 +88,11 @@ message InspectMachineResponse {
message MachineDetails { message MachineDetails {
Metadata metadata = 1; Metadata metadata = 1;
MachineInfo machine = 2; MachineInfo machine = 2;
// Current Corrosion cr-sqlite database version (Lamport timestamp) of the cluster store.
int64 store_db_version = 3;
// Round-trip times to other machines in the cluster, keyed by peer machine ID. // Round-trip times to other machines in the cluster, keyed by peer machine ID.
map<string, RTTStats> rtts = 4; map<string, RTTStats> rtts = 4;
// Current cluster store version observed on this machine.
// Per-actor vector (Corrosion actor UUID → max applied db_version) read from Corrosion crsql_db_versions.
map<string, int64> store_version = 5;
} }
message TokenResponse { message TokenResponse {
+67 -51
View File
@@ -96,7 +96,8 @@ func newClusterController(
func (cc *clusterController) Run(ctx context.Context) error { func (cc *clusterController) Run(ctx context.Context) error {
defer close(cc.stopped) defer close(cc.stopped)
if err := firewall.ConfigureIptablesChains(network.MachineIP(cc.state.Network.Subnet), cc.state.Network.EffectiveWireGuardPort()); err != nil { if err := firewall.ConfigureIptablesChains(network.MachineIP(cc.state.Network.Subnet),
cc.state.Network.EffectiveWireGuardPort()); err != nil {
return fmt.Errorf("configure iptables chains: %w", err) return fmt.Errorf("configure iptables chains: %w", err)
} }
@@ -159,7 +160,9 @@ func (cc *clusterController) Run(ctx context.Context) error {
// Wait for the store database to sync to the minimum version before starting store-dependent components. // Wait for the store database to sync to the minimum version before starting store-dependent components.
// This prevents issues with using partially replicated data when the machine just joined the cluster, // This prevents issues with using partially replicated data when the machine just joined the cluster,
// e.g., an empty machine list causing WireGuard peer misconfiguration. // e.g., an empty machine list causing WireGuard peer misconfiguration.
cc.waitStoreSync(ctx) if err = cc.waitStoreSync(ctx); err != nil {
return fmt.Errorf("wait initial cluster store sync: %w", err)
}
// Check if waitStoreSync exited because the context was cancelled. Return early in that case. // Check if waitStoreSync exited because the context was cancelled. Return early in that case.
if ctx.Err() != nil { if ctx.Err() != nil {
@@ -341,101 +344,114 @@ func (cc *clusterController) handleEndpointChanges(ctx context.Context) {
} }
} }
// waitStoreSync waits for the store database to sync to the minimum required DB version if set in the machine state. // waitStoreSync blocks until the local store version >= state.MinStoreVersion and any known gaps are synced.
// Blocks until synced or context is cancelled. // No-op when MinStoreVersion is empty. Clears state.MinStoreVersion when reached.
func (cc *clusterController) waitStoreSync(ctx context.Context) { func (cc *clusterController) waitStoreSync(ctx context.Context) error {
minVersion := cc.state.MinStoreDBVersion target := cc.state.MinStoreVersion
if minVersion == 0 { if len(target) == 0 {
return return nil
} }
slog.Info("Waiting for the initial cluster store sync.", "min_version", minVersion) slog.Info("Waiting for the initial cluster store sync.", "actors", len(target))
ticker := time.NewTicker(500 * time.Millisecond) ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop() defer ticker.Stop()
// Periodic warning to surface stuck NAT/connectivity issues without aborting.
warnInterval := 5 * time.Minute
warnTimer := time.NewTimer(warnInterval)
defer warnTimer.Stop()
var ( var (
lastVersion int64 lastLagging int
lastLogTime time.Time
lastErrLogTime time.Time lastErrLogTime time.Time
) )
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return return nil
case <-warnTimer.C:
local, err := cc.store.Version(ctx)
if err == nil {
slog.Error("Cluster store sync still pending. Check connectivity to peers.",
"lagging_actors", laggingActors(local, target))
} else {
slog.Error("Cluster store sync still pending. Check connectivity to peers.", "err", err)
}
warnTimer.Reset(warnInterval)
case <-ticker.C: case <-ticker.C:
version, err := cc.store.DBVersion(ctx) local, err := cc.store.Version(ctx)
if err != nil { if err != nil {
// Log errors at most once every 5 seconds. // Throttle error logs to once every 5 seconds.
if time.Since(lastErrLogTime) >= 5*time.Second { if time.Since(lastErrLogTime) >= 5*time.Second {
slog.Error("Failed to get the cluster store DB version, retrying.", "err", err) slog.Error("Failed to get the cluster store version, retrying.", "err", err)
lastErrLogTime = time.Now() lastErrLogTime = time.Now()
} }
continue continue
} }
if version >= minVersion { lagging := laggingActors(local, target)
// Clear MinStoreDBVersion so next restart doesn't wait for sync. if len(lagging) == 0 {
// Per-actor max doesn't imply contiguous apply: corrosion can buffer X:N before
// X:N-1 arrives and track the gap separately. Wait for any remaining gaps to be synced.
if err := cc.waitKnownMissingChanges(ctx); err != nil {
return fmt.Errorf("wait for known missing changes: %w", err)
}
// If the context was cancelled mid-gap-fill, don't persist a "synced" state.
if ctx.Err() != nil {
return nil
}
// Clear MinStoreVersion so next restart doesn't wait for sync.
cc.state.mu.Lock() cc.state.mu.Lock()
cc.state.MinStoreDBVersion = 0 cc.state.MinStoreVersion = nil
if err := cc.state.Save(); err != nil { err = cc.state.Save()
slog.Error("Failed to save machine state after the initial cluster store sync.", "err", err)
}
cc.state.mu.Unlock() cc.state.mu.Unlock()
if err != nil {
// Wait for all known missing changes to be synced before returning. return fmt.Errorf("save machine state after the initial cluster store sync: %w", err)
// TODO: reevaluate if this is necessary after migrating to the latest Corrosion version:
// https://github.com/psviderski/uncloud/issues/172
// This works on the best effort basis as the missing changes may not be yet known when we start
// checking it after reaching the minimum version.
// Reaching the minimum version doesn't guarantee that the store is actually synced to
// the state we observed on the source node. db_version is a machine-local Lamport clock.
// When changes are received from a remote machine, the local db_version is set to
// max(local_db_version, incoming_db_version) + 1 for each applied transaction. This means
// the new machine's db_version can jump well past the source machine's db_version on the very
// first batch of replicated changes, without having received all changes from all machines
// in the cluster.
cc.waitKnownMissingChanges(ctx)
if ver, verErr := cc.store.DBVersion(ctx); verErr == nil {
version = ver
} }
slog.Info("Cluster store completed the initial sync.", "version", version, "min_version", minVersion)
return slog.Info("Cluster store completed the initial sync.", "actors", len(target))
return nil
} }
// Log progress only once a second. if len(lagging) != lastLagging {
if version != lastVersion && time.Since(lastLogTime) >= 1*time.Second { slog.Info("Syncing cluster store.", "lagging_actors", lagging)
slog.Info("Syncing cluster store.", "version", version, "min_version", minVersion) lastLagging = len(lagging)
lastLogTime = time.Now()
lastVersion = version
} }
} }
} }
} }
// laggingActors returns target actors whose local version is below the required value, as [have, need].
func laggingActors(local, target map[string]int64) map[string][2]int64 {
lagging := make(map[string][2]int64)
for actor, need := range target {
if have := local[actor]; have < need {
lagging[actor] = [2]int64{have, need}
}
}
return lagging
}
// waitKnownMissingChanges polls the store until all known missing changes have been synced. // waitKnownMissingChanges polls the store until all known missing changes have been synced.
func (cc *clusterController) waitKnownMissingChanges(ctx context.Context) { func (cc *clusterController) waitKnownMissingChanges(ctx context.Context) error {
ticker := time.NewTicker(1 * time.Second) ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop() defer ticker.Stop()
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return return nil
case <-ticker.C: case <-ticker.C:
changes, err := cc.store.KnownMissingChanges(ctx) changes, err := cc.store.KnownMissingChanges(ctx)
if err != nil { if err != nil {
slog.Error("Failed to get known missing changes from the cluster store, skipping check.", return fmt.Errorf("query known missing changes from cluster store: %w", err)
"err", err)
return
} }
if len(changes) == 0 { if len(changes) == 0 {
slog.Debug("All known missing changes have been synced to the cluster store.") slog.Debug("All known missing changes have been synced to the cluster store.")
return return nil
} }
slog.Debug("Waiting for known missing changes to be synced to the cluster store.", "remaining", slog.Debug("Waiting for known missing changes to be synced to the cluster store.", "remaining",
+5 -5
View File
@@ -841,7 +841,7 @@ func (m *Machine) JoinCluster(_ context.Context, req *pb.JoinClusterRequest) (*e
PrivateKey: m.state.Network.PrivateKey, PrivateKey: m.state.Network.PrivateKey,
PublicKey: m.state.Network.PublicKey, PublicKey: m.state.Network.PublicKey,
} }
m.state.MinStoreDBVersion = req.MinStoreDbVersion m.state.MinStoreVersion = req.MinStoreVersion
// Build a peers config from other cluster machines. // Build a peers config from other cluster machines.
m.state.Network.Peers = make([]network.PeerConfig, 0, len(req.OtherMachines)) m.state.Network.Peers = make([]network.PeerConfig, 0, len(req.OtherMachines))
@@ -928,9 +928,9 @@ func (m *Machine) Inspect(_ context.Context, _ *emptypb.Empty) (*pb.MachineInfo,
} }
func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.InspectMachineResponse, error) { func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.InspectMachineResponse, error) {
dbVersion, err := m.store.DBVersion(ctx) storeVersion, err := m.store.Version(ctx)
if err != nil { if err != nil {
return nil, status.Errorf(codes.Internal, "get database version of the cluster store: %v", err) return nil, status.Errorf(codes.Internal, "get cluster store version: %v", err)
} }
var rtts map[string]*pb.RTTStats var rtts map[string]*pb.RTTStats
@@ -954,8 +954,8 @@ func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.Ins
PublicKey: m.state.Network.PublicKey, PublicKey: m.state.Network.PublicKey,
}, },
}, },
StoreDbVersion: dbVersion, StoreVersion: storeVersion,
Rtts: rtts, Rtts: rtts,
}, },
}, },
}, nil }, nil
+4 -3
View File
@@ -24,9 +24,10 @@ type State struct {
Name string Name string
// Network specifies the network configuration for this machine. // Network specifies the network configuration for this machine.
Network *network.Config Network *network.Config
// MinStoreDBVersion is the latest database version of one of the existing cluster machines at the time this machine // MinStoreVersion is the cluster store version this machine must reach before participating.
// joined the cluster. The machine should sync to at least this version before starting any cluster operations. // Per-actor vector (Corrosion actor UUID → max applied db_version) captured from an existing
MinStoreDBVersion int64 `json:",omitempty"` // member at join time. Cleared once reached.
MinStoreVersion map[string]int64 `json:",omitempty"`
// path is the file path config is read from and saved to. // path is the file path config is read from and saved to.
path string path string
+21 -12
View File
@@ -8,6 +8,7 @@ import (
"fmt" "fmt"
"log/slog" "log/slog"
"github.com/google/uuid"
"github.com/psviderski/uncloud/internal/corrosion" "github.com/psviderski/uncloud/internal/corrosion"
"github.com/psviderski/uncloud/internal/machine/api/pb" "github.com/psviderski/uncloud/internal/machine/api/pb"
"google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/encoding/protojson"
@@ -57,23 +58,31 @@ func (s *Store) Delete(ctx context.Context, key string) error {
return err return err
} }
// DBVersion returns the current cr-sqlite database version (Lamport timestamp). // Version returns the cluster store's per-actor version vector:
func (s *Store) DBVersion(ctx context.Context) (int64, error) { // Corrosion actor ID (UUID string) → max received db_version for that actor.
rows, err := s.corro.QueryContext(ctx, "SELECT crsql_db_version()") func (s *Store) Version(ctx context.Context) (map[string]int64, error) {
rows, err := s.corro.QueryContext(ctx, "SELECT site_id, db_version FROM crsql_db_versions")
if err != nil { if err != nil {
return 0, fmt.Errorf("query crsql_db_version(): %w", err) return nil, fmt.Errorf("query crsql_db_versions: %w", err)
} }
defer rows.Close() defer rows.Close()
if !rows.Next() { versions := make(map[string]int64)
return 0, fmt.Errorf("no result from crsql_db_version()") for rows.Next() {
var (
siteID []byte
version int64
)
if err = rows.Scan(&siteID, &version); err != nil {
return nil, fmt.Errorf("scan actor version: %w", err)
}
actor, err := uuid.FromBytes(siteID)
if err != nil {
return nil, fmt.Errorf("parse site_id as UUID: %w", err)
}
versions[actor.String()] = version
} }
return versions, nil
var version int64
if err = rows.Scan(&version); err != nil {
return 0, fmt.Errorf("scan db version: %w", err)
}
return version, nil
} }
type MissingChange struct { type MissingChange struct {
+3 -3
View File
@@ -192,9 +192,9 @@ func (p *Provisioner) initCluster(ctx context.Context, machines []Machine) error
// Configure the machine to join the cluster. // Configure the machine to join the cluster.
joinReq := &pb.JoinClusterRequest{ joinReq := &pb.JoinClusterRequest{
Machine: addResp.Machine, Machine: addResp.Machine,
OtherMachines: []*pb.MachineInfo{initResp.Machine}, OtherMachines: []*pb.MachineInfo{initResp.Machine},
MinStoreDbVersion: inspectResp.Machines[0].StoreDbVersion, MinStoreVersion: inspectResp.Machines[0].StoreVersion,
} }
if _, err = cli.JoinCluster(ctx, joinReq); err != nil { if _, err = cli.JoinCluster(ctx, joinReq); err != nil {
return fmt.Errorf("join cluster: %w", err) return fmt.Errorf("join cluster: %w", err)