refactor: migrate to latest Corrosion version 2026.5.14 for ucind (compare versions vector for store sync)

This commit is contained in:
Pasha Sviderski
2026-05-25 14:09:51 +10:00
parent 0dd4bc9d36
commit 9236aa341a
8 changed files with 237 additions and 190 deletions
+124 -105
View File
@@ -391,10 +391,11 @@ type JoinClusterRequest struct {
Machine *MachineInfo `protobuf:"bytes,1,opt,name=machine,proto3" json:"machine,omitempty"`
OtherMachines []*MachineInfo `protobuf:"bytes,3,rep,name=other_machines,json=otherMachines,proto3" json:"other_machines,omitempty"`
// Minimum store database version the new machine should sync to before starting cluster operations.
MinStoreDbVersion int64 `protobuf:"varint,4,opt,name=min_store_db_version,json=minStoreDbVersion,proto3" json:"min_store_db_version,omitempty"`
// WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set.
WireguardPort int32 `protobuf:"varint,5,opt,name=wireguard_port,json=wireguardPort,proto3" json:"wireguard_port,omitempty"`
// Cluster store version this machine must reach before participating.
// Per-actor vector (Corrosion actor UUID → max applied db_version).
MinStoreVersion map[string]int64 `protobuf:"bytes,6,rep,name=min_store_version,json=minStoreVersion,proto3" json:"min_store_version,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
}
func (x *JoinClusterRequest) Reset() {
@@ -443,13 +444,6 @@ func (x *JoinClusterRequest) GetOtherMachines() []*MachineInfo {
return nil
}
func (x *JoinClusterRequest) GetMinStoreDbVersion() int64 {
if x != nil {
return x.MinStoreDbVersion
}
return 0
}
func (x *JoinClusterRequest) GetWireguardPort() int32 {
if x != nil {
return x.WireguardPort
@@ -457,6 +451,13 @@ func (x *JoinClusterRequest) GetWireguardPort() int32 {
return 0
}
func (x *JoinClusterRequest) GetMinStoreVersion() map[string]int64 {
if x != nil {
return x.MinStoreVersion
}
return nil
}
type InspectMachineResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -512,10 +513,11 @@ type MachineDetails struct {
Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
Machine *MachineInfo `protobuf:"bytes,2,opt,name=machine,proto3" json:"machine,omitempty"`
// Current Corrosion cr-sqlite database version (Lamport timestamp) of the cluster store.
StoreDbVersion int64 `protobuf:"varint,3,opt,name=store_db_version,json=storeDbVersion,proto3" json:"store_db_version,omitempty"`
// Round-trip times to other machines in the cluster, keyed by peer machine ID.
Rtts map[string]*RTTStats `protobuf:"bytes,4,rep,name=rtts,proto3" json:"rtts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// Current cluster store version observed on this machine.
// Per-actor vector (Corrosion actor UUID → max applied db_version) read from Corrosion crsql_db_versions.
StoreVersion map[string]int64 `protobuf:"bytes,5,rep,name=store_version,json=storeVersion,proto3" json:"store_version,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
}
func (x *MachineDetails) Reset() {
@@ -564,13 +566,6 @@ func (x *MachineDetails) GetMachine() *MachineInfo {
return nil
}
func (x *MachineDetails) GetStoreDbVersion() int64 {
if x != nil {
return x.StoreDbVersion
}
return 0
}
func (x *MachineDetails) GetRtts() map[string]*RTTStats {
if x != nil {
return x.Rtts
@@ -578,6 +573,13 @@ func (x *MachineDetails) GetRtts() map[string]*RTTStats {
return nil
}
func (x *MachineDetails) GetStoreVersion() map[string]int64 {
if x != nil {
return x.StoreVersion
}
return nil
}
type TokenResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -1054,7 +1056,7 @@ type Service_Container struct {
func (x *Service_Container) Reset() {
*x = Service_Container{}
if protoimpl.UnsafeEnabled {
mi := &file_internal_machine_api_pb_machine_proto_msgTypes[17]
mi := &file_internal_machine_api_pb_machine_proto_msgTypes[19]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1067,7 +1069,7 @@ func (x *Service_Container) String() string {
func (*Service_Container) ProtoMessage() {}
func (x *Service_Container) ProtoReflect() protoreflect.Message {
mi := &file_internal_machine_api_pb_machine_proto_msgTypes[17]
mi := &file_internal_machine_api_pb_machine_proto_msgTypes[19]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1158,7 +1160,7 @@ var file_internal_machine_api_pb_machine_proto_rawDesc = []byte{
0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a,
0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10,
0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f,
0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0xd1, 0x01, 0x0a, 0x12, 0x4a, 0x6f,
0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0xbe, 0x02, 0x0a, 0x12, 0x4a, 0x6f,
0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49,
@@ -1166,33 +1168,46 @@ var file_internal_machine_api_pb_machine_proto_rawDesc = []byte{
0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x03,
0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69,
0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x4d, 0x61, 0x63,
0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x74, 0x6f,
0x72, 0x65, 0x5f, 0x64, 0x62, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20,
0x01, 0x28, 0x03, 0x52, 0x11, 0x6d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x44, 0x62, 0x56,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75,
0x61, 0x72, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d,
0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x49, 0x0a,
0x16, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69,
0x6e, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e,
0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x08,
0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x22, 0x8c, 0x02, 0x0a, 0x0e, 0x4d, 0x61, 0x63,
0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x29, 0x0a, 0x08, 0x6d,
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e,
0x61, 0x70, 0x69, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65,
0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e,
0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61,
0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69,
0x6e, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x64, 0x62, 0x5f, 0x76,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x73, 0x74,
0x6f, 0x72, 0x65, 0x44, 0x62, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x04,
0x72, 0x74, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x61, 0x70, 0x69,
0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x2e,
0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x72, 0x74, 0x74, 0x73, 0x1a,
0x46, 0x0a, 0x09, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03,
0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23,
0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e,
0x61, 0x70, 0x69, 0x2e, 0x52, 0x54, 0x54, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61,
0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61,
0x72, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x77,
0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x58, 0x0a, 0x11,
0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
0x6e, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4a, 0x6f,
0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x2e, 0x4d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x6d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x42, 0x0a, 0x14, 0x4d, 0x69, 0x6e, 0x53, 0x74, 0x6f,
0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52,
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x49, 0x0a, 0x16, 0x49, 0x6e,
0x73, 0x70, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73,
0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63,
0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x08, 0x6d, 0x61, 0x63,
0x68, 0x69, 0x6e, 0x65, 0x73, 0x22, 0xef, 0x02, 0x0a, 0x0e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e,
0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x29, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61,
0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61, 0x70, 0x69,
0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64,
0x61, 0x74, 0x61, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x02,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69,
0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12,
0x31, 0x0a, 0x04, 0x72, 0x74, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e,
0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69,
0x6c, 0x73, 0x2e, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x72, 0x74,
0x74, 0x73, 0x12, 0x4a, 0x0a, 0x0d, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73,
0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x2e,
0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x53,
0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79,
0x52, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x46,
0x0a, 0x09, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x0a,
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61,
0x70, 0x69, 0x2e, 0x52, 0x54, 0x54, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c,
0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3f, 0x0a, 0x11, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a,
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61,
0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25, 0x0a, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65,
0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x0e,
@@ -1311,7 +1326,7 @@ func file_internal_machine_api_pb_machine_proto_rawDescGZIP() []byte {
return file_internal_machine_api_pb_machine_proto_rawDescData
}
var file_internal_machine_api_pb_machine_proto_msgTypes = make([]protoimpl.MessageInfo, 18)
var file_internal_machine_api_pb_machine_proto_msgTypes = make([]protoimpl.MessageInfo, 20)
var file_internal_machine_api_pb_machine_proto_goTypes = []any{
(*MachineInfo)(nil), // 0: api.MachineInfo
(*NetworkConfig)(nil), // 1: api.NetworkConfig
@@ -1329,66 +1344,70 @@ var file_internal_machine_api_pb_machine_proto_goTypes = []any{
(*InspectWireGuardNetworkResponse)(nil), // 13: api.InspectWireGuardNetworkResponse
(*WireGuardPeer)(nil), // 14: api.WireGuardPeer
(*RTTStats)(nil), // 15: api.RTTStats
nil, // 16: api.MachineDetails.RttsEntry
(*Service_Container)(nil), // 17: api.Service.Container
(*IP)(nil), // 18: api.IP
(*IPPrefix)(nil), // 19: api.IPPrefix
(*IPPort)(nil), // 20: api.IPPort
(*Metadata)(nil), // 21: api.Metadata
(*timestamppb.Timestamp)(nil), // 22: google.protobuf.Timestamp
(*durationpb.Duration)(nil), // 23: google.protobuf.Duration
(*emptypb.Empty)(nil), // 24: google.protobuf.Empty
(*LogsRequest)(nil), // 25: api.LogsRequest
(*LogEntry)(nil), // 26: api.LogEntry
nil, // 16: api.JoinClusterRequest.MinStoreVersionEntry
nil, // 17: api.MachineDetails.RttsEntry
nil, // 18: api.MachineDetails.StoreVersionEntry
(*Service_Container)(nil), // 19: api.Service.Container
(*IP)(nil), // 20: api.IP
(*IPPrefix)(nil), // 21: api.IPPrefix
(*IPPort)(nil), // 22: api.IPPort
(*Metadata)(nil), // 23: api.Metadata
(*timestamppb.Timestamp)(nil), // 24: google.protobuf.Timestamp
(*durationpb.Duration)(nil), // 25: google.protobuf.Duration
(*emptypb.Empty)(nil), // 26: google.protobuf.Empty
(*LogsRequest)(nil), // 27: api.LogsRequest
(*LogEntry)(nil), // 28: api.LogEntry
}
var file_internal_machine_api_pb_machine_proto_depIdxs = []int32{
1, // 0: api.MachineInfo.network:type_name -> api.NetworkConfig
18, // 1: api.MachineInfo.public_ip:type_name -> api.IP
19, // 2: api.NetworkConfig.subnet:type_name -> api.IPPrefix
18, // 3: api.NetworkConfig.management_ip:type_name -> api.IP
20, // 4: api.NetworkConfig.endpoints:type_name -> api.IPPort
19, // 5: api.InitClusterRequest.network:type_name -> api.IPPrefix
18, // 6: api.InitClusterRequest.public_ip:type_name -> api.IP
20, // 7: api.InitClusterRequest.wireguard_endpoints:type_name -> api.IPPort
20, // 1: api.MachineInfo.public_ip:type_name -> api.IP
21, // 2: api.NetworkConfig.subnet:type_name -> api.IPPrefix
20, // 3: api.NetworkConfig.management_ip:type_name -> api.IP
22, // 4: api.NetworkConfig.endpoints:type_name -> api.IPPort
21, // 5: api.InitClusterRequest.network:type_name -> api.IPPrefix
20, // 6: api.InitClusterRequest.public_ip:type_name -> api.IP
22, // 7: api.InitClusterRequest.wireguard_endpoints:type_name -> api.IPPort
0, // 8: api.InitClusterResponse.machine:type_name -> api.MachineInfo
0, // 9: api.JoinClusterRequest.machine:type_name -> api.MachineInfo
0, // 10: api.JoinClusterRequest.other_machines:type_name -> api.MachineInfo
7, // 11: api.InspectMachineResponse.machines:type_name -> api.MachineDetails
21, // 12: api.MachineDetails.metadata:type_name -> api.Metadata
0, // 13: api.MachineDetails.machine:type_name -> api.MachineInfo
16, // 14: api.MachineDetails.rtts:type_name -> api.MachineDetails.RttsEntry
17, // 15: api.Service.containers:type_name -> api.Service.Container
10, // 16: api.InspectServiceResponse.service:type_name -> api.Service
14, // 17: api.InspectWireGuardNetworkResponse.peers:type_name -> api.WireGuardPeer
22, // 18: api.WireGuardPeer.last_handshake_time:type_name -> google.protobuf.Timestamp
23, // 19: api.RTTStats.median:type_name -> google.protobuf.Duration
23, // 20: api.RTTStats.std_dev:type_name -> google.protobuf.Duration
15, // 21: api.MachineDetails.RttsEntry.value:type_name -> api.RTTStats
24, // 22: api.Machine.CheckPrerequisites:input_type -> google.protobuf.Empty
3, // 23: api.Machine.InitCluster:input_type -> api.InitClusterRequest
5, // 24: api.Machine.JoinCluster:input_type -> api.JoinClusterRequest
24, // 25: api.Machine.Token:input_type -> google.protobuf.Empty
24, // 26: api.Machine.Inspect:input_type -> google.protobuf.Empty
24, // 27: api.Machine.InspectMachine:input_type -> google.protobuf.Empty
24, // 28: api.Machine.InspectWireGuardNetwork:input_type -> google.protobuf.Empty
9, // 29: api.Machine.Reset:input_type -> api.ResetRequest
11, // 30: api.Machine.InspectService:input_type -> api.InspectServiceRequest
25, // 31: api.Machine.MachineLogs:input_type -> api.LogsRequest
2, // 32: api.Machine.CheckPrerequisites:output_type -> api.CheckPrerequisitesResponse
4, // 33: api.Machine.InitCluster:output_type -> api.InitClusterResponse
24, // 34: api.Machine.JoinCluster:output_type -> google.protobuf.Empty
8, // 35: api.Machine.Token:output_type -> api.TokenResponse
0, // 36: api.Machine.Inspect:output_type -> api.MachineInfo
6, // 37: api.Machine.InspectMachine:output_type -> api.InspectMachineResponse
13, // 38: api.Machine.InspectWireGuardNetwork:output_type -> api.InspectWireGuardNetworkResponse
24, // 39: api.Machine.Reset:output_type -> google.protobuf.Empty
12, // 40: api.Machine.InspectService:output_type -> api.InspectServiceResponse
26, // 41: api.Machine.MachineLogs:output_type -> api.LogEntry
32, // [32:42] is the sub-list for method output_type
22, // [22:32] is the sub-list for method input_type
22, // [22:22] is the sub-list for extension type_name
22, // [22:22] is the sub-list for extension extendee
0, // [0:22] is the sub-list for field type_name
16, // 11: api.JoinClusterRequest.min_store_version:type_name -> api.JoinClusterRequest.MinStoreVersionEntry
7, // 12: api.InspectMachineResponse.machines:type_name -> api.MachineDetails
23, // 13: api.MachineDetails.metadata:type_name -> api.Metadata
0, // 14: api.MachineDetails.machine:type_name -> api.MachineInfo
17, // 15: api.MachineDetails.rtts:type_name -> api.MachineDetails.RttsEntry
18, // 16: api.MachineDetails.store_version:type_name -> api.MachineDetails.StoreVersionEntry
19, // 17: api.Service.containers:type_name -> api.Service.Container
10, // 18: api.InspectServiceResponse.service:type_name -> api.Service
14, // 19: api.InspectWireGuardNetworkResponse.peers:type_name -> api.WireGuardPeer
24, // 20: api.WireGuardPeer.last_handshake_time:type_name -> google.protobuf.Timestamp
25, // 21: api.RTTStats.median:type_name -> google.protobuf.Duration
25, // 22: api.RTTStats.std_dev:type_name -> google.protobuf.Duration
15, // 23: api.MachineDetails.RttsEntry.value:type_name -> api.RTTStats
26, // 24: api.Machine.CheckPrerequisites:input_type -> google.protobuf.Empty
3, // 25: api.Machine.InitCluster:input_type -> api.InitClusterRequest
5, // 26: api.Machine.JoinCluster:input_type -> api.JoinClusterRequest
26, // 27: api.Machine.Token:input_type -> google.protobuf.Empty
26, // 28: api.Machine.Inspect:input_type -> google.protobuf.Empty
26, // 29: api.Machine.InspectMachine:input_type -> google.protobuf.Empty
26, // 30: api.Machine.InspectWireGuardNetwork:input_type -> google.protobuf.Empty
9, // 31: api.Machine.Reset:input_type -> api.ResetRequest
11, // 32: api.Machine.InspectService:input_type -> api.InspectServiceRequest
27, // 33: api.Machine.MachineLogs:input_type -> api.LogsRequest
2, // 34: api.Machine.CheckPrerequisites:output_type -> api.CheckPrerequisitesResponse
4, // 35: api.Machine.InitCluster:output_type -> api.InitClusterResponse
26, // 36: api.Machine.JoinCluster:output_type -> google.protobuf.Empty
8, // 37: api.Machine.Token:output_type -> api.TokenResponse
0, // 38: api.Machine.Inspect:output_type -> api.MachineInfo
6, // 39: api.Machine.InspectMachine:output_type -> api.InspectMachineResponse
13, // 40: api.Machine.InspectWireGuardNetwork:output_type -> api.InspectWireGuardNetworkResponse
26, // 41: api.Machine.Reset:output_type -> google.protobuf.Empty
12, // 42: api.Machine.InspectService:output_type -> api.InspectServiceResponse
28, // 43: api.Machine.MachineLogs:output_type -> api.LogEntry
34, // [34:44] is the sub-list for method output_type
24, // [24:34] is the sub-list for method input_type
24, // [24:24] is the sub-list for extension type_name
24, // [24:24] is the sub-list for extension extendee
0, // [0:24] is the sub-list for field type_name
}
func init() { file_internal_machine_api_pb_machine_proto_init() }
@@ -1590,7 +1609,7 @@ func file_internal_machine_api_pb_machine_proto_init() {
return nil
}
}
file_internal_machine_api_pb_machine_proto_msgTypes[17].Exporter = func(v any, i int) any {
file_internal_machine_api_pb_machine_proto_msgTypes[19].Exporter = func(v any, i int) any {
switch v := v.(*Service_Container); i {
case 0:
return &v.state
@@ -1613,7 +1632,7 @@ func file_internal_machine_api_pb_machine_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_internal_machine_api_pb_machine_proto_rawDesc,
NumEnums: 0,
NumMessages: 18,
NumMessages: 20,
NumExtensions: 0,
NumServices: 1,
},
+6 -4
View File
@@ -73,10 +73,11 @@ message InitClusterResponse {
message JoinClusterRequest {
MachineInfo machine = 1;
repeated MachineInfo other_machines = 3;
// Minimum store database version the new machine should sync to before starting cluster operations.
int64 min_store_db_version = 4;
// WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set.
int32 wireguard_port = 5;
// Cluster store version this machine must reach before participating.
// Per-actor vector (Corrosion actor UUID → max applied db_version).
map<string, int64> min_store_version = 6;
}
message InspectMachineResponse {
@@ -87,10 +88,11 @@ message InspectMachineResponse {
message MachineDetails {
Metadata metadata = 1;
MachineInfo machine = 2;
// Current Corrosion cr-sqlite database version (Lamport timestamp) of the cluster store.
int64 store_db_version = 3;
// Round-trip times to other machines in the cluster, keyed by peer machine ID.
map<string, RTTStats> rtts = 4;
// Current cluster store version observed on this machine.
// Per-actor vector (Corrosion actor UUID → max applied db_version) read from Corrosion crsql_db_versions.
map<string, int64> store_version = 5;
}
message TokenResponse {
+67 -51
View File
@@ -96,7 +96,8 @@ func newClusterController(
func (cc *clusterController) Run(ctx context.Context) error {
defer close(cc.stopped)
if err := firewall.ConfigureIptablesChains(network.MachineIP(cc.state.Network.Subnet), cc.state.Network.EffectiveWireGuardPort()); err != nil {
if err := firewall.ConfigureIptablesChains(network.MachineIP(cc.state.Network.Subnet),
cc.state.Network.EffectiveWireGuardPort()); err != nil {
return fmt.Errorf("configure iptables chains: %w", err)
}
@@ -159,7 +160,9 @@ func (cc *clusterController) Run(ctx context.Context) error {
// Wait for the store database to sync to the minimum version before starting store-dependent components.
// This prevents issues with using partially replicated data when the machine just joined the cluster,
// e.g., an empty machine list causing WireGuard peer misconfiguration.
cc.waitStoreSync(ctx)
if err = cc.waitStoreSync(ctx); err != nil {
return fmt.Errorf("wait initial cluster store sync: %w", err)
}
// Check if waitStoreSync exited because the context was cancelled. Return early in that case.
if ctx.Err() != nil {
@@ -341,101 +344,114 @@ func (cc *clusterController) handleEndpointChanges(ctx context.Context) {
}
}
// waitStoreSync waits for the store database to sync to the minimum required DB version if set in the machine state.
// Blocks until synced or context is cancelled.
func (cc *clusterController) waitStoreSync(ctx context.Context) {
minVersion := cc.state.MinStoreDBVersion
if minVersion == 0 {
return
// waitStoreSync blocks until the local store version >= state.MinStoreVersion and any known gaps are synced.
// No-op when MinStoreVersion is empty. Clears state.MinStoreVersion when reached.
func (cc *clusterController) waitStoreSync(ctx context.Context) error {
target := cc.state.MinStoreVersion
if len(target) == 0 {
return nil
}
slog.Info("Waiting for the initial cluster store sync.", "min_version", minVersion)
slog.Info("Waiting for the initial cluster store sync.", "actors", len(target))
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
// Periodic warning to surface stuck NAT/connectivity issues without aborting.
warnInterval := 5 * time.Minute
warnTimer := time.NewTimer(warnInterval)
defer warnTimer.Stop()
var (
lastVersion int64
lastLogTime time.Time
lastLagging int
lastErrLogTime time.Time
)
for {
select {
case <-ctx.Done():
return
return nil
case <-warnTimer.C:
local, err := cc.store.Version(ctx)
if err == nil {
slog.Error("Cluster store sync still pending. Check connectivity to peers.",
"lagging_actors", laggingActors(local, target))
} else {
slog.Error("Cluster store sync still pending. Check connectivity to peers.", "err", err)
}
warnTimer.Reset(warnInterval)
case <-ticker.C:
version, err := cc.store.DBVersion(ctx)
local, err := cc.store.Version(ctx)
if err != nil {
// Log errors at most once every 5 seconds.
// Throttle error logs to once every 5 seconds.
if time.Since(lastErrLogTime) >= 5*time.Second {
slog.Error("Failed to get the cluster store DB version, retrying.", "err", err)
slog.Error("Failed to get the cluster store version, retrying.", "err", err)
lastErrLogTime = time.Now()
}
continue
}
if version >= minVersion {
// Clear MinStoreDBVersion so next restart doesn't wait for sync.
lagging := laggingActors(local, target)
if len(lagging) == 0 {
// Per-actor max doesn't imply contiguous apply: corrosion can buffer X:N before
// X:N-1 arrives and track the gap separately. Wait for any remaining gaps to be synced.
if err := cc.waitKnownMissingChanges(ctx); err != nil {
return fmt.Errorf("wait for known missing changes: %w", err)
}
// If the context was cancelled mid-gap-fill, don't persist a "synced" state.
if ctx.Err() != nil {
return nil
}
// Clear MinStoreVersion so next restart doesn't wait for sync.
cc.state.mu.Lock()
cc.state.MinStoreDBVersion = 0
if err := cc.state.Save(); err != nil {
slog.Error("Failed to save machine state after the initial cluster store sync.", "err", err)
}
cc.state.MinStoreVersion = nil
err = cc.state.Save()
cc.state.mu.Unlock()
// Wait for all known missing changes to be synced before returning.
// TODO: reevaluate if this is necessary after migrating to the latest Corrosion version:
// https://github.com/psviderski/uncloud/issues/172
// This works on the best effort basis as the missing changes may not be yet known when we start
// checking it after reaching the minimum version.
// Reaching the minimum version doesn't guarantee that the store is actually synced to
// the state we observed on the source node. db_version is a machine-local Lamport clock.
// When changes are received from a remote machine, the local db_version is set to
// max(local_db_version, incoming_db_version) + 1 for each applied transaction. This means
// the new machine's db_version can jump well past the source machine's db_version on the very
// first batch of replicated changes, without having received all changes from all machines
// in the cluster.
cc.waitKnownMissingChanges(ctx)
if ver, verErr := cc.store.DBVersion(ctx); verErr == nil {
version = ver
if err != nil {
return fmt.Errorf("save machine state after the initial cluster store sync: %w", err)
}
slog.Info("Cluster store completed the initial sync.", "version", version, "min_version", minVersion)
return
slog.Info("Cluster store completed the initial sync.", "actors", len(target))
return nil
}
// Log progress only once a second.
if version != lastVersion && time.Since(lastLogTime) >= 1*time.Second {
slog.Info("Syncing cluster store.", "version", version, "min_version", minVersion)
lastLogTime = time.Now()
lastVersion = version
if len(lagging) != lastLagging {
slog.Info("Syncing cluster store.", "lagging_actors", lagging)
lastLagging = len(lagging)
}
}
}
}
// laggingActors returns target actors whose local version is below the required value, as [have, need].
func laggingActors(local, target map[string]int64) map[string][2]int64 {
lagging := make(map[string][2]int64)
for actor, need := range target {
if have := local[actor]; have < need {
lagging[actor] = [2]int64{have, need}
}
}
return lagging
}
// waitKnownMissingChanges polls the store until all known missing changes have been synced.
func (cc *clusterController) waitKnownMissingChanges(ctx context.Context) {
func (cc *clusterController) waitKnownMissingChanges(ctx context.Context) error {
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
return nil
case <-ticker.C:
changes, err := cc.store.KnownMissingChanges(ctx)
if err != nil {
slog.Error("Failed to get known missing changes from the cluster store, skipping check.",
"err", err)
return
return fmt.Errorf("query known missing changes from cluster store: %w", err)
}
if len(changes) == 0 {
slog.Debug("All known missing changes have been synced to the cluster store.")
return
return nil
}
slog.Debug("Waiting for known missing changes to be synced to the cluster store.", "remaining",
+5 -5
View File
@@ -841,7 +841,7 @@ func (m *Machine) JoinCluster(_ context.Context, req *pb.JoinClusterRequest) (*e
PrivateKey: m.state.Network.PrivateKey,
PublicKey: m.state.Network.PublicKey,
}
m.state.MinStoreDBVersion = req.MinStoreDbVersion
m.state.MinStoreVersion = req.MinStoreVersion
// Build a peers config from other cluster machines.
m.state.Network.Peers = make([]network.PeerConfig, 0, len(req.OtherMachines))
@@ -928,9 +928,9 @@ func (m *Machine) Inspect(_ context.Context, _ *emptypb.Empty) (*pb.MachineInfo,
}
func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.InspectMachineResponse, error) {
dbVersion, err := m.store.DBVersion(ctx)
storeVersion, err := m.store.Version(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "get database version of the cluster store: %v", err)
return nil, status.Errorf(codes.Internal, "get cluster store version: %v", err)
}
var rtts map[string]*pb.RTTStats
@@ -954,8 +954,8 @@ func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.Ins
PublicKey: m.state.Network.PublicKey,
},
},
StoreDbVersion: dbVersion,
Rtts: rtts,
StoreVersion: storeVersion,
Rtts: rtts,
},
},
}, nil
+4 -3
View File
@@ -24,9 +24,10 @@ type State struct {
Name string
// Network specifies the network configuration for this machine.
Network *network.Config
// MinStoreDBVersion is the latest database version of one of the existing cluster machines at the time this machine
// joined the cluster. The machine should sync to at least this version before starting any cluster operations.
MinStoreDBVersion int64 `json:",omitempty"`
// MinStoreVersion is the cluster store version this machine must reach before participating.
// Per-actor vector (Corrosion actor UUID → max applied db_version) captured from an existing
// member at join time. Cleared once reached.
MinStoreVersion map[string]int64 `json:",omitempty"`
// path is the file path config is read from and saved to.
path string
+21 -12
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"log/slog"
"github.com/google/uuid"
"github.com/psviderski/uncloud/internal/corrosion"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"google.golang.org/protobuf/encoding/protojson"
@@ -57,23 +58,31 @@ func (s *Store) Delete(ctx context.Context, key string) error {
return err
}
// DBVersion returns the current cr-sqlite database version (Lamport timestamp).
func (s *Store) DBVersion(ctx context.Context) (int64, error) {
rows, err := s.corro.QueryContext(ctx, "SELECT crsql_db_version()")
// Version returns the cluster store's per-actor version vector:
// Corrosion actor ID (UUID string) → max received db_version for that actor.
func (s *Store) Version(ctx context.Context) (map[string]int64, error) {
rows, err := s.corro.QueryContext(ctx, "SELECT site_id, db_version FROM crsql_db_versions")
if err != nil {
return 0, fmt.Errorf("query crsql_db_version(): %w", err)
return nil, fmt.Errorf("query crsql_db_versions: %w", err)
}
defer rows.Close()
if !rows.Next() {
return 0, fmt.Errorf("no result from crsql_db_version()")
versions := make(map[string]int64)
for rows.Next() {
var (
siteID []byte
version int64
)
if err = rows.Scan(&siteID, &version); err != nil {
return nil, fmt.Errorf("scan actor version: %w", err)
}
actor, err := uuid.FromBytes(siteID)
if err != nil {
return nil, fmt.Errorf("parse site_id as UUID: %w", err)
}
versions[actor.String()] = version
}
var version int64
if err = rows.Scan(&version); err != nil {
return 0, fmt.Errorf("scan db version: %w", err)
}
return version, nil
return versions, nil
}
type MissingChange struct {