mirror of
https://github.com/psviderski/uncloud.git
synced 2026-08-26 11:03:34 +00:00
refactor: migrate to latest Corrosion version 2026.5.14 for ucind (compare versions vector for store sync)
This commit is contained in:
+4
-4
@@ -434,8 +434,8 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
|
|||||||
return nil, nil, fmt.Errorf("add machine to cluster (context '%s'): %w", contextName, err)
|
return nil, nil, fmt.Errorf("add machine to cluster (context '%s'): %w", contextName, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the current store DB version from the cluster to pass to the join request.
|
// Snapshot the cluster store version so the new machine can catch up before participating.
|
||||||
var storeDBVersion int64
|
var storeVersion map[string]int64
|
||||||
inspectResp, err := c.MachineClient.InspectMachine(ctx, &emptypb.Empty{})
|
inspectResp, err := c.MachineClient.InspectMachine(ctx, &emptypb.Empty{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// TODO(lhf): remove Unimplemented check when v0.17.0 is released.
|
// TODO(lhf): remove Unimplemented check when v0.17.0 is released.
|
||||||
@@ -443,7 +443,7 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
|
|||||||
return nil, nil, fmt.Errorf("inspect current cluster machine: %w", err)
|
return nil, nil, fmt.Errorf("inspect current cluster machine: %w", err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
storeDBVersion = inspectResp.Machines[0].StoreDbVersion
|
storeVersion = inspectResp.Machines[0].StoreVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the most up-to-date list of other machines in the cluster to include them in the join request.
|
// Get the most up-to-date list of other machines in the cluster to include them in the join request.
|
||||||
@@ -462,7 +462,7 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (*client
|
|||||||
joinReq := &pb.JoinClusterRequest{
|
joinReq := &pb.JoinClusterRequest{
|
||||||
Machine: addResp.Machine,
|
Machine: addResp.Machine,
|
||||||
OtherMachines: otherMachines,
|
OtherMachines: otherMachines,
|
||||||
MinStoreDbVersion: storeDBVersion,
|
MinStoreVersion: storeVersion,
|
||||||
WireguardPort: int32(opts.WireguardPort),
|
WireguardPort: int32(opts.WireguardPort),
|
||||||
}
|
}
|
||||||
if _, err = machineClient.JoinCluster(ctx, joinReq); err != nil {
|
if _, err = machineClient.JoinCluster(ctx, joinReq); err != nil {
|
||||||
|
|||||||
@@ -391,10 +391,11 @@ type JoinClusterRequest struct {
|
|||||||
|
|
||||||
Machine *MachineInfo `protobuf:"bytes,1,opt,name=machine,proto3" json:"machine,omitempty"`
|
Machine *MachineInfo `protobuf:"bytes,1,opt,name=machine,proto3" json:"machine,omitempty"`
|
||||||
OtherMachines []*MachineInfo `protobuf:"bytes,3,rep,name=other_machines,json=otherMachines,proto3" json:"other_machines,omitempty"`
|
OtherMachines []*MachineInfo `protobuf:"bytes,3,rep,name=other_machines,json=otherMachines,proto3" json:"other_machines,omitempty"`
|
||||||
// Minimum store database version the new machine should sync to before starting cluster operations.
|
|
||||||
MinStoreDbVersion int64 `protobuf:"varint,4,opt,name=min_store_db_version,json=minStoreDbVersion,proto3" json:"min_store_db_version,omitempty"`
|
|
||||||
// WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set.
|
// WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set.
|
||||||
WireguardPort int32 `protobuf:"varint,5,opt,name=wireguard_port,json=wireguardPort,proto3" json:"wireguard_port,omitempty"`
|
WireguardPort int32 `protobuf:"varint,5,opt,name=wireguard_port,json=wireguardPort,proto3" json:"wireguard_port,omitempty"`
|
||||||
|
// Cluster store version this machine must reach before participating.
|
||||||
|
// Per-actor vector (Corrosion actor UUID → max applied db_version).
|
||||||
|
MinStoreVersion map[string]int64 `protobuf:"bytes,6,rep,name=min_store_version,json=minStoreVersion,proto3" json:"min_store_version,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *JoinClusterRequest) Reset() {
|
func (x *JoinClusterRequest) Reset() {
|
||||||
@@ -443,13 +444,6 @@ func (x *JoinClusterRequest) GetOtherMachines() []*MachineInfo {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *JoinClusterRequest) GetMinStoreDbVersion() int64 {
|
|
||||||
if x != nil {
|
|
||||||
return x.MinStoreDbVersion
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *JoinClusterRequest) GetWireguardPort() int32 {
|
func (x *JoinClusterRequest) GetWireguardPort() int32 {
|
||||||
if x != nil {
|
if x != nil {
|
||||||
return x.WireguardPort
|
return x.WireguardPort
|
||||||
@@ -457,6 +451,13 @@ func (x *JoinClusterRequest) GetWireguardPort() int32 {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (x *JoinClusterRequest) GetMinStoreVersion() map[string]int64 {
|
||||||
|
if x != nil {
|
||||||
|
return x.MinStoreVersion
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type InspectMachineResponse struct {
|
type InspectMachineResponse struct {
|
||||||
state protoimpl.MessageState
|
state protoimpl.MessageState
|
||||||
sizeCache protoimpl.SizeCache
|
sizeCache protoimpl.SizeCache
|
||||||
@@ -512,10 +513,11 @@ type MachineDetails struct {
|
|||||||
|
|
||||||
Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
|
Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
|
||||||
Machine *MachineInfo `protobuf:"bytes,2,opt,name=machine,proto3" json:"machine,omitempty"`
|
Machine *MachineInfo `protobuf:"bytes,2,opt,name=machine,proto3" json:"machine,omitempty"`
|
||||||
// Current Corrosion cr-sqlite database version (Lamport timestamp) of the cluster store.
|
|
||||||
StoreDbVersion int64 `protobuf:"varint,3,opt,name=store_db_version,json=storeDbVersion,proto3" json:"store_db_version,omitempty"`
|
|
||||||
// Round-trip times to other machines in the cluster, keyed by peer machine ID.
|
// Round-trip times to other machines in the cluster, keyed by peer machine ID.
|
||||||
Rtts map[string]*RTTStats `protobuf:"bytes,4,rep,name=rtts,proto3" json:"rtts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
|
Rtts map[string]*RTTStats `protobuf:"bytes,4,rep,name=rtts,proto3" json:"rtts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
|
||||||
|
// Current cluster store version observed on this machine.
|
||||||
|
// Per-actor vector (Corrosion actor UUID → max applied db_version) read from Corrosion crsql_db_versions.
|
||||||
|
StoreVersion map[string]int64 `protobuf:"bytes,5,rep,name=store_version,json=storeVersion,proto3" json:"store_version,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *MachineDetails) Reset() {
|
func (x *MachineDetails) Reset() {
|
||||||
@@ -564,13 +566,6 @@ func (x *MachineDetails) GetMachine() *MachineInfo {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *MachineDetails) GetStoreDbVersion() int64 {
|
|
||||||
if x != nil {
|
|
||||||
return x.StoreDbVersion
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MachineDetails) GetRtts() map[string]*RTTStats {
|
func (x *MachineDetails) GetRtts() map[string]*RTTStats {
|
||||||
if x != nil {
|
if x != nil {
|
||||||
return x.Rtts
|
return x.Rtts
|
||||||
@@ -578,6 +573,13 @@ func (x *MachineDetails) GetRtts() map[string]*RTTStats {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (x *MachineDetails) GetStoreVersion() map[string]int64 {
|
||||||
|
if x != nil {
|
||||||
|
return x.StoreVersion
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type TokenResponse struct {
|
type TokenResponse struct {
|
||||||
state protoimpl.MessageState
|
state protoimpl.MessageState
|
||||||
sizeCache protoimpl.SizeCache
|
sizeCache protoimpl.SizeCache
|
||||||
@@ -1054,7 +1056,7 @@ type Service_Container struct {
|
|||||||
func (x *Service_Container) Reset() {
|
func (x *Service_Container) Reset() {
|
||||||
*x = Service_Container{}
|
*x = Service_Container{}
|
||||||
if protoimpl.UnsafeEnabled {
|
if protoimpl.UnsafeEnabled {
|
||||||
mi := &file_internal_machine_api_pb_machine_proto_msgTypes[17]
|
mi := &file_internal_machine_api_pb_machine_proto_msgTypes[19]
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
ms.StoreMessageInfo(mi)
|
ms.StoreMessageInfo(mi)
|
||||||
}
|
}
|
||||||
@@ -1067,7 +1069,7 @@ func (x *Service_Container) String() string {
|
|||||||
func (*Service_Container) ProtoMessage() {}
|
func (*Service_Container) ProtoMessage() {}
|
||||||
|
|
||||||
func (x *Service_Container) ProtoReflect() protoreflect.Message {
|
func (x *Service_Container) ProtoReflect() protoreflect.Message {
|
||||||
mi := &file_internal_machine_api_pb_machine_proto_msgTypes[17]
|
mi := &file_internal_machine_api_pb_machine_proto_msgTypes[19]
|
||||||
if protoimpl.UnsafeEnabled && x != nil {
|
if protoimpl.UnsafeEnabled && x != nil {
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
if ms.LoadMessageInfo() == nil {
|
if ms.LoadMessageInfo() == nil {
|
||||||
@@ -1158,7 +1160,7 @@ var file_internal_machine_api_pb_machine_proto_rawDesc = []byte{
|
|||||||
0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a,
|
0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a,
|
||||||
0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10,
|
0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10,
|
||||||
0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f,
|
0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f,
|
||||||
0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0xd1, 0x01, 0x0a, 0x12, 0x4a, 0x6f,
|
0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0xbe, 0x02, 0x0a, 0x12, 0x4a, 0x6f,
|
||||||
0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
||||||
0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
|
0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
|
||||||
0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49,
|
0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49,
|
||||||
@@ -1166,33 +1168,46 @@ var file_internal_machine_api_pb_machine_proto_rawDesc = []byte{
|
|||||||
0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x03,
|
0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x03,
|
||||||
0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69,
|
0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69,
|
||||||
0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x4d, 0x61, 0x63,
|
0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x4d, 0x61, 0x63,
|
||||||
0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x74, 0x6f,
|
0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61,
|
||||||
0x72, 0x65, 0x5f, 0x64, 0x62, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20,
|
0x72, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x77,
|
||||||
0x01, 0x28, 0x03, 0x52, 0x11, 0x6d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x44, 0x62, 0x56,
|
0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x58, 0x0a, 0x11,
|
||||||
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75,
|
0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
|
||||||
0x61, 0x72, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d,
|
0x6e, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4a, 0x6f,
|
||||||
0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x49, 0x0a,
|
0x69, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
||||||
0x16, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52,
|
0x2e, 0x4d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
|
||||||
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69,
|
0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x6d, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56,
|
||||||
0x6e, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e,
|
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x42, 0x0a, 0x14, 0x4d, 0x69, 0x6e, 0x53, 0x74, 0x6f,
|
||||||
0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x08,
|
0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
|
||||||
0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x22, 0x8c, 0x02, 0x0a, 0x0e, 0x4d, 0x61, 0x63,
|
0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
|
||||||
0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x29, 0x0a, 0x08, 0x6d,
|
0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52,
|
||||||
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e,
|
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x49, 0x0a, 0x16, 0x49, 0x6e,
|
||||||
0x61, 0x70, 0x69, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65,
|
0x73, 0x70, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70,
|
||||||
0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e,
|
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73,
|
||||||
0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61,
|
0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63,
|
||||||
0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69,
|
0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x08, 0x6d, 0x61, 0x63,
|
||||||
0x6e, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x64, 0x62, 0x5f, 0x76,
|
0x68, 0x69, 0x6e, 0x65, 0x73, 0x22, 0xef, 0x02, 0x0a, 0x0e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e,
|
||||||
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x73, 0x74,
|
0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x29, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61,
|
||||||
0x6f, 0x72, 0x65, 0x44, 0x62, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x04,
|
0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61, 0x70, 0x69,
|
||||||
0x72, 0x74, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x61, 0x70, 0x69,
|
0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64,
|
||||||
0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x2e,
|
0x61, 0x74, 0x61, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x02,
|
||||||
0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x72, 0x74, 0x74, 0x73, 0x1a,
|
0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69,
|
||||||
0x46, 0x0a, 0x09, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03,
|
0x6e, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12,
|
||||||
0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23,
|
0x31, 0x0a, 0x04, 0x72, 0x74, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e,
|
||||||
0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e,
|
0x61, 0x70, 0x69, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69,
|
||||||
0x61, 0x70, 0x69, 0x2e, 0x52, 0x54, 0x54, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61,
|
0x6c, 0x73, 0x2e, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x72, 0x74,
|
||||||
|
0x74, 0x73, 0x12, 0x4a, 0x0a, 0x0d, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73,
|
||||||
|
0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x2e,
|
||||||
|
0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x53,
|
||||||
|
0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79,
|
||||||
|
0x52, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x46,
|
||||||
|
0x0a, 0x09, 0x52, 0x74, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
|
||||||
|
0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x0a,
|
||||||
|
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x61,
|
||||||
|
0x70, 0x69, 0x2e, 0x52, 0x54, 0x54, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c,
|
||||||
|
0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3f, 0x0a, 0x11, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x56,
|
||||||
|
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
|
||||||
|
0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a,
|
||||||
|
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61,
|
||||||
0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25, 0x0a, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
|
0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25, 0x0a, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
|
||||||
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65,
|
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65,
|
||||||
0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x0e,
|
0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x0e,
|
||||||
@@ -1311,7 +1326,7 @@ func file_internal_machine_api_pb_machine_proto_rawDescGZIP() []byte {
|
|||||||
return file_internal_machine_api_pb_machine_proto_rawDescData
|
return file_internal_machine_api_pb_machine_proto_rawDescData
|
||||||
}
|
}
|
||||||
|
|
||||||
var file_internal_machine_api_pb_machine_proto_msgTypes = make([]protoimpl.MessageInfo, 18)
|
var file_internal_machine_api_pb_machine_proto_msgTypes = make([]protoimpl.MessageInfo, 20)
|
||||||
var file_internal_machine_api_pb_machine_proto_goTypes = []any{
|
var file_internal_machine_api_pb_machine_proto_goTypes = []any{
|
||||||
(*MachineInfo)(nil), // 0: api.MachineInfo
|
(*MachineInfo)(nil), // 0: api.MachineInfo
|
||||||
(*NetworkConfig)(nil), // 1: api.NetworkConfig
|
(*NetworkConfig)(nil), // 1: api.NetworkConfig
|
||||||
@@ -1329,66 +1344,70 @@ var file_internal_machine_api_pb_machine_proto_goTypes = []any{
|
|||||||
(*InspectWireGuardNetworkResponse)(nil), // 13: api.InspectWireGuardNetworkResponse
|
(*InspectWireGuardNetworkResponse)(nil), // 13: api.InspectWireGuardNetworkResponse
|
||||||
(*WireGuardPeer)(nil), // 14: api.WireGuardPeer
|
(*WireGuardPeer)(nil), // 14: api.WireGuardPeer
|
||||||
(*RTTStats)(nil), // 15: api.RTTStats
|
(*RTTStats)(nil), // 15: api.RTTStats
|
||||||
nil, // 16: api.MachineDetails.RttsEntry
|
nil, // 16: api.JoinClusterRequest.MinStoreVersionEntry
|
||||||
(*Service_Container)(nil), // 17: api.Service.Container
|
nil, // 17: api.MachineDetails.RttsEntry
|
||||||
(*IP)(nil), // 18: api.IP
|
nil, // 18: api.MachineDetails.StoreVersionEntry
|
||||||
(*IPPrefix)(nil), // 19: api.IPPrefix
|
(*Service_Container)(nil), // 19: api.Service.Container
|
||||||
(*IPPort)(nil), // 20: api.IPPort
|
(*IP)(nil), // 20: api.IP
|
||||||
(*Metadata)(nil), // 21: api.Metadata
|
(*IPPrefix)(nil), // 21: api.IPPrefix
|
||||||
(*timestamppb.Timestamp)(nil), // 22: google.protobuf.Timestamp
|
(*IPPort)(nil), // 22: api.IPPort
|
||||||
(*durationpb.Duration)(nil), // 23: google.protobuf.Duration
|
(*Metadata)(nil), // 23: api.Metadata
|
||||||
(*emptypb.Empty)(nil), // 24: google.protobuf.Empty
|
(*timestamppb.Timestamp)(nil), // 24: google.protobuf.Timestamp
|
||||||
(*LogsRequest)(nil), // 25: api.LogsRequest
|
(*durationpb.Duration)(nil), // 25: google.protobuf.Duration
|
||||||
(*LogEntry)(nil), // 26: api.LogEntry
|
(*emptypb.Empty)(nil), // 26: google.protobuf.Empty
|
||||||
|
(*LogsRequest)(nil), // 27: api.LogsRequest
|
||||||
|
(*LogEntry)(nil), // 28: api.LogEntry
|
||||||
}
|
}
|
||||||
var file_internal_machine_api_pb_machine_proto_depIdxs = []int32{
|
var file_internal_machine_api_pb_machine_proto_depIdxs = []int32{
|
||||||
1, // 0: api.MachineInfo.network:type_name -> api.NetworkConfig
|
1, // 0: api.MachineInfo.network:type_name -> api.NetworkConfig
|
||||||
18, // 1: api.MachineInfo.public_ip:type_name -> api.IP
|
20, // 1: api.MachineInfo.public_ip:type_name -> api.IP
|
||||||
19, // 2: api.NetworkConfig.subnet:type_name -> api.IPPrefix
|
21, // 2: api.NetworkConfig.subnet:type_name -> api.IPPrefix
|
||||||
18, // 3: api.NetworkConfig.management_ip:type_name -> api.IP
|
20, // 3: api.NetworkConfig.management_ip:type_name -> api.IP
|
||||||
20, // 4: api.NetworkConfig.endpoints:type_name -> api.IPPort
|
22, // 4: api.NetworkConfig.endpoints:type_name -> api.IPPort
|
||||||
19, // 5: api.InitClusterRequest.network:type_name -> api.IPPrefix
|
21, // 5: api.InitClusterRequest.network:type_name -> api.IPPrefix
|
||||||
18, // 6: api.InitClusterRequest.public_ip:type_name -> api.IP
|
20, // 6: api.InitClusterRequest.public_ip:type_name -> api.IP
|
||||||
20, // 7: api.InitClusterRequest.wireguard_endpoints:type_name -> api.IPPort
|
22, // 7: api.InitClusterRequest.wireguard_endpoints:type_name -> api.IPPort
|
||||||
0, // 8: api.InitClusterResponse.machine:type_name -> api.MachineInfo
|
0, // 8: api.InitClusterResponse.machine:type_name -> api.MachineInfo
|
||||||
0, // 9: api.JoinClusterRequest.machine:type_name -> api.MachineInfo
|
0, // 9: api.JoinClusterRequest.machine:type_name -> api.MachineInfo
|
||||||
0, // 10: api.JoinClusterRequest.other_machines:type_name -> api.MachineInfo
|
0, // 10: api.JoinClusterRequest.other_machines:type_name -> api.MachineInfo
|
||||||
7, // 11: api.InspectMachineResponse.machines:type_name -> api.MachineDetails
|
16, // 11: api.JoinClusterRequest.min_store_version:type_name -> api.JoinClusterRequest.MinStoreVersionEntry
|
||||||
21, // 12: api.MachineDetails.metadata:type_name -> api.Metadata
|
7, // 12: api.InspectMachineResponse.machines:type_name -> api.MachineDetails
|
||||||
0, // 13: api.MachineDetails.machine:type_name -> api.MachineInfo
|
23, // 13: api.MachineDetails.metadata:type_name -> api.Metadata
|
||||||
16, // 14: api.MachineDetails.rtts:type_name -> api.MachineDetails.RttsEntry
|
0, // 14: api.MachineDetails.machine:type_name -> api.MachineInfo
|
||||||
17, // 15: api.Service.containers:type_name -> api.Service.Container
|
17, // 15: api.MachineDetails.rtts:type_name -> api.MachineDetails.RttsEntry
|
||||||
10, // 16: api.InspectServiceResponse.service:type_name -> api.Service
|
18, // 16: api.MachineDetails.store_version:type_name -> api.MachineDetails.StoreVersionEntry
|
||||||
14, // 17: api.InspectWireGuardNetworkResponse.peers:type_name -> api.WireGuardPeer
|
19, // 17: api.Service.containers:type_name -> api.Service.Container
|
||||||
22, // 18: api.WireGuardPeer.last_handshake_time:type_name -> google.protobuf.Timestamp
|
10, // 18: api.InspectServiceResponse.service:type_name -> api.Service
|
||||||
23, // 19: api.RTTStats.median:type_name -> google.protobuf.Duration
|
14, // 19: api.InspectWireGuardNetworkResponse.peers:type_name -> api.WireGuardPeer
|
||||||
23, // 20: api.RTTStats.std_dev:type_name -> google.protobuf.Duration
|
24, // 20: api.WireGuardPeer.last_handshake_time:type_name -> google.protobuf.Timestamp
|
||||||
15, // 21: api.MachineDetails.RttsEntry.value:type_name -> api.RTTStats
|
25, // 21: api.RTTStats.median:type_name -> google.protobuf.Duration
|
||||||
24, // 22: api.Machine.CheckPrerequisites:input_type -> google.protobuf.Empty
|
25, // 22: api.RTTStats.std_dev:type_name -> google.protobuf.Duration
|
||||||
3, // 23: api.Machine.InitCluster:input_type -> api.InitClusterRequest
|
15, // 23: api.MachineDetails.RttsEntry.value:type_name -> api.RTTStats
|
||||||
5, // 24: api.Machine.JoinCluster:input_type -> api.JoinClusterRequest
|
26, // 24: api.Machine.CheckPrerequisites:input_type -> google.protobuf.Empty
|
||||||
24, // 25: api.Machine.Token:input_type -> google.protobuf.Empty
|
3, // 25: api.Machine.InitCluster:input_type -> api.InitClusterRequest
|
||||||
24, // 26: api.Machine.Inspect:input_type -> google.protobuf.Empty
|
5, // 26: api.Machine.JoinCluster:input_type -> api.JoinClusterRequest
|
||||||
24, // 27: api.Machine.InspectMachine:input_type -> google.protobuf.Empty
|
26, // 27: api.Machine.Token:input_type -> google.protobuf.Empty
|
||||||
24, // 28: api.Machine.InspectWireGuardNetwork:input_type -> google.protobuf.Empty
|
26, // 28: api.Machine.Inspect:input_type -> google.protobuf.Empty
|
||||||
9, // 29: api.Machine.Reset:input_type -> api.ResetRequest
|
26, // 29: api.Machine.InspectMachine:input_type -> google.protobuf.Empty
|
||||||
11, // 30: api.Machine.InspectService:input_type -> api.InspectServiceRequest
|
26, // 30: api.Machine.InspectWireGuardNetwork:input_type -> google.protobuf.Empty
|
||||||
25, // 31: api.Machine.MachineLogs:input_type -> api.LogsRequest
|
9, // 31: api.Machine.Reset:input_type -> api.ResetRequest
|
||||||
2, // 32: api.Machine.CheckPrerequisites:output_type -> api.CheckPrerequisitesResponse
|
11, // 32: api.Machine.InspectService:input_type -> api.InspectServiceRequest
|
||||||
4, // 33: api.Machine.InitCluster:output_type -> api.InitClusterResponse
|
27, // 33: api.Machine.MachineLogs:input_type -> api.LogsRequest
|
||||||
24, // 34: api.Machine.JoinCluster:output_type -> google.protobuf.Empty
|
2, // 34: api.Machine.CheckPrerequisites:output_type -> api.CheckPrerequisitesResponse
|
||||||
8, // 35: api.Machine.Token:output_type -> api.TokenResponse
|
4, // 35: api.Machine.InitCluster:output_type -> api.InitClusterResponse
|
||||||
0, // 36: api.Machine.Inspect:output_type -> api.MachineInfo
|
26, // 36: api.Machine.JoinCluster:output_type -> google.protobuf.Empty
|
||||||
6, // 37: api.Machine.InspectMachine:output_type -> api.InspectMachineResponse
|
8, // 37: api.Machine.Token:output_type -> api.TokenResponse
|
||||||
13, // 38: api.Machine.InspectWireGuardNetwork:output_type -> api.InspectWireGuardNetworkResponse
|
0, // 38: api.Machine.Inspect:output_type -> api.MachineInfo
|
||||||
24, // 39: api.Machine.Reset:output_type -> google.protobuf.Empty
|
6, // 39: api.Machine.InspectMachine:output_type -> api.InspectMachineResponse
|
||||||
12, // 40: api.Machine.InspectService:output_type -> api.InspectServiceResponse
|
13, // 40: api.Machine.InspectWireGuardNetwork:output_type -> api.InspectWireGuardNetworkResponse
|
||||||
26, // 41: api.Machine.MachineLogs:output_type -> api.LogEntry
|
26, // 41: api.Machine.Reset:output_type -> google.protobuf.Empty
|
||||||
32, // [32:42] is the sub-list for method output_type
|
12, // 42: api.Machine.InspectService:output_type -> api.InspectServiceResponse
|
||||||
22, // [22:32] is the sub-list for method input_type
|
28, // 43: api.Machine.MachineLogs:output_type -> api.LogEntry
|
||||||
22, // [22:22] is the sub-list for extension type_name
|
34, // [34:44] is the sub-list for method output_type
|
||||||
22, // [22:22] is the sub-list for extension extendee
|
24, // [24:34] is the sub-list for method input_type
|
||||||
0, // [0:22] is the sub-list for field type_name
|
24, // [24:24] is the sub-list for extension type_name
|
||||||
|
24, // [24:24] is the sub-list for extension extendee
|
||||||
|
0, // [0:24] is the sub-list for field type_name
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() { file_internal_machine_api_pb_machine_proto_init() }
|
func init() { file_internal_machine_api_pb_machine_proto_init() }
|
||||||
@@ -1590,7 +1609,7 @@ func file_internal_machine_api_pb_machine_proto_init() {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
file_internal_machine_api_pb_machine_proto_msgTypes[17].Exporter = func(v any, i int) any {
|
file_internal_machine_api_pb_machine_proto_msgTypes[19].Exporter = func(v any, i int) any {
|
||||||
switch v := v.(*Service_Container); i {
|
switch v := v.(*Service_Container); i {
|
||||||
case 0:
|
case 0:
|
||||||
return &v.state
|
return &v.state
|
||||||
@@ -1613,7 +1632,7 @@ func file_internal_machine_api_pb_machine_proto_init() {
|
|||||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||||
RawDescriptor: file_internal_machine_api_pb_machine_proto_rawDesc,
|
RawDescriptor: file_internal_machine_api_pb_machine_proto_rawDesc,
|
||||||
NumEnums: 0,
|
NumEnums: 0,
|
||||||
NumMessages: 18,
|
NumMessages: 20,
|
||||||
NumExtensions: 0,
|
NumExtensions: 0,
|
||||||
NumServices: 1,
|
NumServices: 1,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -73,10 +73,11 @@ message InitClusterResponse {
|
|||||||
message JoinClusterRequest {
|
message JoinClusterRequest {
|
||||||
MachineInfo machine = 1;
|
MachineInfo machine = 1;
|
||||||
repeated MachineInfo other_machines = 3;
|
repeated MachineInfo other_machines = 3;
|
||||||
// Minimum store database version the new machine should sync to before starting cluster operations.
|
|
||||||
int64 min_store_db_version = 4;
|
|
||||||
// WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set.
|
// WireGuard listen port for this machine. Uses the default port (51820) if 0 or not set.
|
||||||
int32 wireguard_port = 5;
|
int32 wireguard_port = 5;
|
||||||
|
// Cluster store version this machine must reach before participating.
|
||||||
|
// Per-actor vector (Corrosion actor UUID → max applied db_version).
|
||||||
|
map<string, int64> min_store_version = 6;
|
||||||
}
|
}
|
||||||
|
|
||||||
message InspectMachineResponse {
|
message InspectMachineResponse {
|
||||||
@@ -87,10 +88,11 @@ message InspectMachineResponse {
|
|||||||
message MachineDetails {
|
message MachineDetails {
|
||||||
Metadata metadata = 1;
|
Metadata metadata = 1;
|
||||||
MachineInfo machine = 2;
|
MachineInfo machine = 2;
|
||||||
// Current Corrosion cr-sqlite database version (Lamport timestamp) of the cluster store.
|
|
||||||
int64 store_db_version = 3;
|
|
||||||
// Round-trip times to other machines in the cluster, keyed by peer machine ID.
|
// Round-trip times to other machines in the cluster, keyed by peer machine ID.
|
||||||
map<string, RTTStats> rtts = 4;
|
map<string, RTTStats> rtts = 4;
|
||||||
|
// Current cluster store version observed on this machine.
|
||||||
|
// Per-actor vector (Corrosion actor UUID → max applied db_version) read from Corrosion crsql_db_versions.
|
||||||
|
map<string, int64> store_version = 5;
|
||||||
}
|
}
|
||||||
|
|
||||||
message TokenResponse {
|
message TokenResponse {
|
||||||
|
|||||||
+69
-53
@@ -96,7 +96,8 @@ func newClusterController(
|
|||||||
func (cc *clusterController) Run(ctx context.Context) error {
|
func (cc *clusterController) Run(ctx context.Context) error {
|
||||||
defer close(cc.stopped)
|
defer close(cc.stopped)
|
||||||
|
|
||||||
if err := firewall.ConfigureIptablesChains(network.MachineIP(cc.state.Network.Subnet), cc.state.Network.EffectiveWireGuardPort()); err != nil {
|
if err := firewall.ConfigureIptablesChains(network.MachineIP(cc.state.Network.Subnet),
|
||||||
|
cc.state.Network.EffectiveWireGuardPort()); err != nil {
|
||||||
return fmt.Errorf("configure iptables chains: %w", err)
|
return fmt.Errorf("configure iptables chains: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -159,7 +160,9 @@ func (cc *clusterController) Run(ctx context.Context) error {
|
|||||||
// Wait for the store database to sync to the minimum version before starting store-dependent components.
|
// Wait for the store database to sync to the minimum version before starting store-dependent components.
|
||||||
// This prevents issues with using partially replicated data when the machine just joined the cluster,
|
// This prevents issues with using partially replicated data when the machine just joined the cluster,
|
||||||
// e.g., an empty machine list causing WireGuard peer misconfiguration.
|
// e.g., an empty machine list causing WireGuard peer misconfiguration.
|
||||||
cc.waitStoreSync(ctx)
|
if err = cc.waitStoreSync(ctx); err != nil {
|
||||||
|
return fmt.Errorf("wait initial cluster store sync: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Check if waitStoreSync exited because the context was cancelled. Return early in that case.
|
// Check if waitStoreSync exited because the context was cancelled. Return early in that case.
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
@@ -341,101 +344,114 @@ func (cc *clusterController) handleEndpointChanges(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitStoreSync waits for the store database to sync to the minimum required DB version if set in the machine state.
|
// waitStoreSync blocks until the local store version >= state.MinStoreVersion and any known gaps are synced.
|
||||||
// Blocks until synced or context is cancelled.
|
// No-op when MinStoreVersion is empty. Clears state.MinStoreVersion when reached.
|
||||||
func (cc *clusterController) waitStoreSync(ctx context.Context) {
|
func (cc *clusterController) waitStoreSync(ctx context.Context) error {
|
||||||
minVersion := cc.state.MinStoreDBVersion
|
target := cc.state.MinStoreVersion
|
||||||
if minVersion == 0 {
|
if len(target) == 0 {
|
||||||
return
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
slog.Info("Waiting for the initial cluster store sync.", "min_version", minVersion)
|
slog.Info("Waiting for the initial cluster store sync.", "actors", len(target))
|
||||||
|
|
||||||
ticker := time.NewTicker(500 * time.Millisecond)
|
ticker := time.NewTicker(500 * time.Millisecond)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
// Periodic warning to surface stuck NAT/connectivity issues without aborting.
|
||||||
|
warnInterval := 5 * time.Minute
|
||||||
|
warnTimer := time.NewTimer(warnInterval)
|
||||||
|
defer warnTimer.Stop()
|
||||||
|
|
||||||
var (
|
var (
|
||||||
lastVersion int64
|
lastLagging int
|
||||||
lastLogTime time.Time
|
|
||||||
lastErrLogTime time.Time
|
lastErrLogTime time.Time
|
||||||
)
|
)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return nil
|
||||||
|
case <-warnTimer.C:
|
||||||
|
local, err := cc.store.Version(ctx)
|
||||||
|
if err == nil {
|
||||||
|
slog.Error("Cluster store sync still pending. Check connectivity to peers.",
|
||||||
|
"lagging_actors", laggingActors(local, target))
|
||||||
|
} else {
|
||||||
|
slog.Error("Cluster store sync still pending. Check connectivity to peers.", "err", err)
|
||||||
|
}
|
||||||
|
warnTimer.Reset(warnInterval)
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
version, err := cc.store.DBVersion(ctx)
|
local, err := cc.store.Version(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Log errors at most once every 5 seconds.
|
// Throttle error logs to once every 5 seconds.
|
||||||
if time.Since(lastErrLogTime) >= 5*time.Second {
|
if time.Since(lastErrLogTime) >= 5*time.Second {
|
||||||
slog.Error("Failed to get the cluster store DB version, retrying.", "err", err)
|
slog.Error("Failed to get the cluster store version, retrying.", "err", err)
|
||||||
lastErrLogTime = time.Now()
|
lastErrLogTime = time.Now()
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if version >= minVersion {
|
lagging := laggingActors(local, target)
|
||||||
// Clear MinStoreDBVersion so next restart doesn't wait for sync.
|
if len(lagging) == 0 {
|
||||||
|
// Per-actor max doesn't imply contiguous apply: corrosion can buffer X:N before
|
||||||
|
// X:N-1 arrives and track the gap separately. Wait for any remaining gaps to be synced.
|
||||||
|
if err := cc.waitKnownMissingChanges(ctx); err != nil {
|
||||||
|
return fmt.Errorf("wait for known missing changes: %w", err)
|
||||||
|
}
|
||||||
|
// If the context was cancelled mid-gap-fill, don't persist a "synced" state.
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear MinStoreVersion so next restart doesn't wait for sync.
|
||||||
cc.state.mu.Lock()
|
cc.state.mu.Lock()
|
||||||
cc.state.MinStoreDBVersion = 0
|
cc.state.MinStoreVersion = nil
|
||||||
if err := cc.state.Save(); err != nil {
|
err = cc.state.Save()
|
||||||
slog.Error("Failed to save machine state after the initial cluster store sync.", "err", err)
|
|
||||||
}
|
|
||||||
cc.state.mu.Unlock()
|
cc.state.mu.Unlock()
|
||||||
|
if err != nil {
|
||||||
// Wait for all known missing changes to be synced before returning.
|
return fmt.Errorf("save machine state after the initial cluster store sync: %w", err)
|
||||||
// TODO: reevaluate if this is necessary after migrating to the latest Corrosion version:
|
|
||||||
// https://github.com/psviderski/uncloud/issues/172
|
|
||||||
// This works on the best effort basis as the missing changes may not be yet known when we start
|
|
||||||
// checking it after reaching the minimum version.
|
|
||||||
// Reaching the minimum version doesn't guarantee that the store is actually synced to
|
|
||||||
// the state we observed on the source node. db_version is a machine-local Lamport clock.
|
|
||||||
// When changes are received from a remote machine, the local db_version is set to
|
|
||||||
// max(local_db_version, incoming_db_version) + 1 for each applied transaction. This means
|
|
||||||
// the new machine's db_version can jump well past the source machine's db_version on the very
|
|
||||||
// first batch of replicated changes, without having received all changes from all machines
|
|
||||||
// in the cluster.
|
|
||||||
cc.waitKnownMissingChanges(ctx)
|
|
||||||
|
|
||||||
if ver, verErr := cc.store.DBVersion(ctx); verErr == nil {
|
|
||||||
version = ver
|
|
||||||
}
|
|
||||||
slog.Info("Cluster store completed the initial sync.", "version", version, "min_version", minVersion)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log progress only once a second.
|
slog.Info("Cluster store completed the initial sync.", "actors", len(target))
|
||||||
if version != lastVersion && time.Since(lastLogTime) >= 1*time.Second {
|
return nil
|
||||||
slog.Info("Syncing cluster store.", "version", version, "min_version", minVersion)
|
}
|
||||||
lastLogTime = time.Now()
|
|
||||||
lastVersion = version
|
if len(lagging) != lastLagging {
|
||||||
|
slog.Info("Syncing cluster store.", "lagging_actors", lagging)
|
||||||
|
lastLagging = len(lagging)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// laggingActors returns target actors whose local version is below the required value, as [have, need].
|
||||||
|
func laggingActors(local, target map[string]int64) map[string][2]int64 {
|
||||||
|
lagging := make(map[string][2]int64)
|
||||||
|
for actor, need := range target {
|
||||||
|
if have := local[actor]; have < need {
|
||||||
|
lagging[actor] = [2]int64{have, need}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return lagging
|
||||||
|
}
|
||||||
|
|
||||||
// waitKnownMissingChanges polls the store until all known missing changes have been synced.
|
// waitKnownMissingChanges polls the store until all known missing changes have been synced.
|
||||||
func (cc *clusterController) waitKnownMissingChanges(ctx context.Context) {
|
func (cc *clusterController) waitKnownMissingChanges(ctx context.Context) error {
|
||||||
ticker := time.NewTicker(1 * time.Second)
|
ticker := time.NewTicker(1 * time.Second)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return nil
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
changes, err := cc.store.KnownMissingChanges(ctx)
|
changes, err := cc.store.KnownMissingChanges(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("Failed to get known missing changes from the cluster store, skipping check.",
|
return fmt.Errorf("query known missing changes from cluster store: %w", err)
|
||||||
"err", err)
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(changes) == 0 {
|
if len(changes) == 0 {
|
||||||
slog.Debug("All known missing changes have been synced to the cluster store.")
|
slog.Debug("All known missing changes have been synced to the cluster store.")
|
||||||
return
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
slog.Debug("Waiting for known missing changes to be synced to the cluster store.", "remaining",
|
slog.Debug("Waiting for known missing changes to be synced to the cluster store.", "remaining",
|
||||||
|
|||||||
@@ -841,7 +841,7 @@ func (m *Machine) JoinCluster(_ context.Context, req *pb.JoinClusterRequest) (*e
|
|||||||
PrivateKey: m.state.Network.PrivateKey,
|
PrivateKey: m.state.Network.PrivateKey,
|
||||||
PublicKey: m.state.Network.PublicKey,
|
PublicKey: m.state.Network.PublicKey,
|
||||||
}
|
}
|
||||||
m.state.MinStoreDBVersion = req.MinStoreDbVersion
|
m.state.MinStoreVersion = req.MinStoreVersion
|
||||||
|
|
||||||
// Build a peers config from other cluster machines.
|
// Build a peers config from other cluster machines.
|
||||||
m.state.Network.Peers = make([]network.PeerConfig, 0, len(req.OtherMachines))
|
m.state.Network.Peers = make([]network.PeerConfig, 0, len(req.OtherMachines))
|
||||||
@@ -928,9 +928,9 @@ func (m *Machine) Inspect(_ context.Context, _ *emptypb.Empty) (*pb.MachineInfo,
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.InspectMachineResponse, error) {
|
func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.InspectMachineResponse, error) {
|
||||||
dbVersion, err := m.store.DBVersion(ctx)
|
storeVersion, err := m.store.Version(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, status.Errorf(codes.Internal, "get database version of the cluster store: %v", err)
|
return nil, status.Errorf(codes.Internal, "get cluster store version: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var rtts map[string]*pb.RTTStats
|
var rtts map[string]*pb.RTTStats
|
||||||
@@ -954,7 +954,7 @@ func (m *Machine) InspectMachine(ctx context.Context, _ *emptypb.Empty) (*pb.Ins
|
|||||||
PublicKey: m.state.Network.PublicKey,
|
PublicKey: m.state.Network.PublicKey,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
StoreDbVersion: dbVersion,
|
StoreVersion: storeVersion,
|
||||||
Rtts: rtts,
|
Rtts: rtts,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -24,9 +24,10 @@ type State struct {
|
|||||||
Name string
|
Name string
|
||||||
// Network specifies the network configuration for this machine.
|
// Network specifies the network configuration for this machine.
|
||||||
Network *network.Config
|
Network *network.Config
|
||||||
// MinStoreDBVersion is the latest database version of one of the existing cluster machines at the time this machine
|
// MinStoreVersion is the cluster store version this machine must reach before participating.
|
||||||
// joined the cluster. The machine should sync to at least this version before starting any cluster operations.
|
// Per-actor vector (Corrosion actor UUID → max applied db_version) captured from an existing
|
||||||
MinStoreDBVersion int64 `json:",omitempty"`
|
// member at join time. Cleared once reached.
|
||||||
|
MinStoreVersion map[string]int64 `json:",omitempty"`
|
||||||
|
|
||||||
// path is the file path config is read from and saved to.
|
// path is the file path config is read from and saved to.
|
||||||
path string
|
path string
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
"github.com/psviderski/uncloud/internal/corrosion"
|
"github.com/psviderski/uncloud/internal/corrosion"
|
||||||
"github.com/psviderski/uncloud/internal/machine/api/pb"
|
"github.com/psviderski/uncloud/internal/machine/api/pb"
|
||||||
"google.golang.org/protobuf/encoding/protojson"
|
"google.golang.org/protobuf/encoding/protojson"
|
||||||
@@ -57,23 +58,31 @@ func (s *Store) Delete(ctx context.Context, key string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// DBVersion returns the current cr-sqlite database version (Lamport timestamp).
|
// Version returns the cluster store's per-actor version vector:
|
||||||
func (s *Store) DBVersion(ctx context.Context) (int64, error) {
|
// Corrosion actor ID (UUID string) → max received db_version for that actor.
|
||||||
rows, err := s.corro.QueryContext(ctx, "SELECT crsql_db_version()")
|
func (s *Store) Version(ctx context.Context) (map[string]int64, error) {
|
||||||
|
rows, err := s.corro.QueryContext(ctx, "SELECT site_id, db_version FROM crsql_db_versions")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("query crsql_db_version(): %w", err)
|
return nil, fmt.Errorf("query crsql_db_versions: %w", err)
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
|
|
||||||
if !rows.Next() {
|
versions := make(map[string]int64)
|
||||||
return 0, fmt.Errorf("no result from crsql_db_version()")
|
for rows.Next() {
|
||||||
|
var (
|
||||||
|
siteID []byte
|
||||||
|
version int64
|
||||||
|
)
|
||||||
|
if err = rows.Scan(&siteID, &version); err != nil {
|
||||||
|
return nil, fmt.Errorf("scan actor version: %w", err)
|
||||||
}
|
}
|
||||||
|
actor, err := uuid.FromBytes(siteID)
|
||||||
var version int64
|
if err != nil {
|
||||||
if err = rows.Scan(&version); err != nil {
|
return nil, fmt.Errorf("parse site_id as UUID: %w", err)
|
||||||
return 0, fmt.Errorf("scan db version: %w", err)
|
|
||||||
}
|
}
|
||||||
return version, nil
|
versions[actor.String()] = version
|
||||||
|
}
|
||||||
|
return versions, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type MissingChange struct {
|
type MissingChange struct {
|
||||||
|
|||||||
@@ -194,7 +194,7 @@ func (p *Provisioner) initCluster(ctx context.Context, machines []Machine) error
|
|||||||
joinReq := &pb.JoinClusterRequest{
|
joinReq := &pb.JoinClusterRequest{
|
||||||
Machine: addResp.Machine,
|
Machine: addResp.Machine,
|
||||||
OtherMachines: []*pb.MachineInfo{initResp.Machine},
|
OtherMachines: []*pb.MachineInfo{initResp.Machine},
|
||||||
MinStoreDbVersion: inspectResp.Machines[0].StoreDbVersion,
|
MinStoreVersion: inspectResp.Machines[0].StoreVersion,
|
||||||
}
|
}
|
||||||
if _, err = cli.JoinCluster(ctx, joinReq); err != nil {
|
if _, err = cli.JoinCluster(ctx, joinReq); err != nil {
|
||||||
return fmt.Errorf("join cluster: %w", err)
|
return fmt.Errorf("join cluster: %w", err)
|
||||||
|
|||||||
Reference in New Issue
Block a user