mirror of
https://github.com/psviderski/uncloud.git
synced 2026-08-26 11:03:34 +00:00
relax relationship between corrosion service, init cluster with new store
This commit is contained in:
@@ -3,6 +3,7 @@ package cluster
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"google.golang.org/grpc/codes"
|
"google.golang.org/grpc/codes"
|
||||||
"google.golang.org/grpc/status"
|
"google.golang.org/grpc/status"
|
||||||
@@ -10,6 +11,7 @@ import (
|
|||||||
"google.golang.org/protobuf/types/known/emptypb"
|
"google.golang.org/protobuf/types/known/emptypb"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
|
"time"
|
||||||
"uncloud/internal/machine/api/pb"
|
"uncloud/internal/machine/api/pb"
|
||||||
"uncloud/internal/machine/network"
|
"uncloud/internal/machine/network"
|
||||||
"uncloud/internal/machine/store"
|
"uncloud/internal/machine/store"
|
||||||
@@ -34,6 +36,35 @@ func NewCluster(state *State, store *store.Store) *Cluster {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Cluster) Init(ctx context.Context, network netip.Prefix) error {
|
||||||
|
initialised, err := c.Initialised(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if initialised {
|
||||||
|
return fmt.Errorf("cluster already initialized")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = c.store.Put(ctx, "network", network.String()); err != nil {
|
||||||
|
return fmt.Errorf("put network to store: %w", err)
|
||||||
|
}
|
||||||
|
if err = c.store.Put(ctx, "created_at", time.Now().UTC().Format(time.RFC3339)); err != nil {
|
||||||
|
return fmt.Errorf("put created_at to store: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cluster) Initialised(ctx context.Context) (bool, error) {
|
||||||
|
var createdAt string
|
||||||
|
if err := c.store.Get(ctx, "created_at", &createdAt); err != nil {
|
||||||
|
if errors.Is(err, store.ErrKeyNotFound) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("get created_at from store: %w", err)
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Cluster) SetState(state *State) {
|
func (c *Cluster) SetState(state *State) {
|
||||||
c.state = state
|
c.state = state
|
||||||
}
|
}
|
||||||
|
|||||||
+26
-22
@@ -144,7 +144,7 @@ func NewMachine(config *Config) (*Machine, error) {
|
|||||||
}
|
}
|
||||||
m.localServer = newGRPCServer(m, c)
|
m.localServer = newGRPCServer(m, c)
|
||||||
|
|
||||||
if m.IsInitialised() {
|
if m.Initialised() {
|
||||||
m.initialised <- struct{}{}
|
m.initialised <- struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -163,9 +163,9 @@ func (m *Machine) Started() <-chan struct{} {
|
|||||||
return m.started
|
return m.started
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsInitialised returns true if the machine has been configured as a member of a cluster,
|
// Initialised returns true if the machine has been configured as a member of a cluster,
|
||||||
// either by initialising a new cluster on it or joining an existing one.
|
// either by initialising a new cluster on it or joining an existing one.
|
||||||
func (m *Machine) IsInitialised() bool {
|
func (m *Machine) Initialised() bool {
|
||||||
m.state.mu.RLock()
|
m.state.mu.RLock()
|
||||||
defer m.state.mu.RUnlock()
|
defer m.state.mu.RUnlock()
|
||||||
|
|
||||||
@@ -173,6 +173,19 @@ func (m *Machine) IsInitialised() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *Machine) Run(ctx context.Context) error {
|
func (m *Machine) Run(ctx context.Context) error {
|
||||||
|
// Configure and start the corrosion service on the loopback if the machine is not initialised as a cluster
|
||||||
|
// member. This provides the store required for the machine to initialise a new cluster on it.
|
||||||
|
if !m.Initialised() {
|
||||||
|
if err := m.configureCorrosion(); err != nil {
|
||||||
|
return fmt.Errorf("configure corrosion service: %w", err)
|
||||||
|
}
|
||||||
|
slog.Info("Configured corrosion service.", "dir", m.config.CorrosionDir)
|
||||||
|
|
||||||
|
if err := m.config.CorrosionService.Start(); err != nil {
|
||||||
|
return fmt.Errorf("start corrosion service: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Use an errgroup to coordinate error handling and graceful shutdown of multiple machine components.
|
// Use an errgroup to coordinate error handling and graceful shutdown of multiple machine components.
|
||||||
errGroup, ctx := errgroup.WithContext(ctx)
|
errGroup, ctx := errgroup.WithContext(ctx)
|
||||||
|
|
||||||
@@ -192,28 +205,10 @@ func (m *Machine) Run(ctx context.Context) error {
|
|||||||
)
|
)
|
||||||
close(m.started)
|
close(m.started)
|
||||||
|
|
||||||
// Configure and start the corrosion service on the loopback if the machine is not initialised as a cluster
|
|
||||||
// member. This provides the store required for the machine to initialise a new cluster on it.
|
|
||||||
if !m.IsInitialised() {
|
|
||||||
// Needs to run in a goroutine because the corrosion systemd service depends on the readiness of the daemon
|
|
||||||
// indicated by closing the started channel.
|
|
||||||
errGroup.Go(func() error {
|
|
||||||
if err := m.configureCorrosion(); err != nil {
|
|
||||||
return fmt.Errorf("configure corrosion service: %w", err)
|
|
||||||
}
|
|
||||||
slog.Info("Configured corrosion service.", "dir", m.config.CorrosionDir)
|
|
||||||
|
|
||||||
if err := m.config.CorrosionService.Start(); err != nil {
|
|
||||||
return fmt.Errorf("start corrosion service: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Control loop for managing the network controller.
|
// Control loop for managing the network controller.
|
||||||
errGroup.Go(
|
errGroup.Go(
|
||||||
func() error {
|
func() error {
|
||||||
if !m.IsInitialised() {
|
if !m.Initialised() {
|
||||||
slog.Info(
|
slog.Info(
|
||||||
"Waiting for the machine to be initialised as a member of a cluster " +
|
"Waiting for the machine to be initialised as a member of a cluster " +
|
||||||
"to start the network controller.",
|
"to start the network controller.",
|
||||||
@@ -386,6 +381,15 @@ func (m *Machine) InitCluster(ctx context.Context, req *pb.InitClusterRequest) (
|
|||||||
return nil, status.Errorf(codes.InvalidArgument, "set cluster network: %v", err)
|
return nil, status.Errorf(codes.InvalidArgument, "set cluster network: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
clusterNetwork, err := req.Network.ToPrefix()
|
||||||
|
if err != nil {
|
||||||
|
return nil, status.Errorf(codes.InvalidArgument, "invalid network: %v", err)
|
||||||
|
}
|
||||||
|
if err = m.cluster.Init(ctx, clusterNetwork); err != nil {
|
||||||
|
return nil, status.Errorf(codes.Internal, "init cluster: %v", err)
|
||||||
|
}
|
||||||
|
slog.Info("Cluster initialised.", "network", clusterNetwork.String())
|
||||||
|
|
||||||
// Use the public and all routable IPs as endpoints.
|
// Use the public and all routable IPs as endpoints.
|
||||||
ips, err := network.ListRoutableIPs()
|
ips, err := network.ListRoutableIPs()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -1,3 +1,10 @@
|
|||||||
|
-- cluster table stores the key-value pairs of the cluster configuration.
|
||||||
|
CREATE TABLE cluster
|
||||||
|
(
|
||||||
|
key TEXT NOT NULL PRIMARY KEY,
|
||||||
|
value ANY
|
||||||
|
);
|
||||||
|
|
||||||
CREATE TABLE machines
|
CREATE TABLE machines
|
||||||
(
|
(
|
||||||
id TEXT NOT NULL PRIMARY KEY,
|
id TEXT NOT NULL PRIMARY KEY,
|
||||||
|
|||||||
@@ -1,14 +1,20 @@
|
|||||||
package store
|
package store
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
_ "embed"
|
_ "embed"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"uncloud/internal/corrosion"
|
"uncloud/internal/corrosion"
|
||||||
"uncloud/internal/machine/api/pb"
|
"uncloud/internal/machine/api/pb"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
//go:embed schema.sql
|
//go:embed schema.sql
|
||||||
var Schema string
|
Schema string
|
||||||
|
|
||||||
|
ErrKeyNotFound = errors.New("key not found")
|
||||||
|
)
|
||||||
|
|
||||||
// Store is a cluster store backed by a distributed Corrosion database.
|
// Store is a cluster store backed by a distributed Corrosion database.
|
||||||
type Store struct {
|
type Store struct {
|
||||||
@@ -19,6 +25,28 @@ func New(corro *corrosion.APIClient) *Store {
|
|||||||
return &Store{corro: corro}
|
return &Store{corro: corro}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Store) Get(ctx context.Context, key string, value any) error {
|
||||||
|
rows, err := s.corro.QueryContext(ctx, "SELECT value FROM cluster WHERE key = ?", key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !rows.Next() {
|
||||||
|
if rows.Err() != nil {
|
||||||
|
return rows.Err()
|
||||||
|
}
|
||||||
|
return ErrKeyNotFound
|
||||||
|
}
|
||||||
|
if err = rows.Scan(value); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) Put(ctx context.Context, key string, value any) error {
|
||||||
|
_, err := s.corro.ExecContext(ctx, "INSERT OR REPLACE INTO cluster (key, value) VALUES (?, ?)", key, value)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Store) CreateMachine(machine *pb.MachineInfo) error {
|
func (s *Store) CreateMachine(machine *pb.MachineInfo) error {
|
||||||
return fmt.Errorf("not implemented")
|
return fmt.Errorf("not implemented")
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-2
@@ -177,8 +177,7 @@ install_corrosion_systemd() {
|
|||||||
cat > "${corrosion_service_path}" << EOF
|
cat > "${corrosion_service_path}" << EOF
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=Uncloud gossip-based distributed store
|
Description=Uncloud gossip-based distributed store
|
||||||
After=uncloud.service
|
PartOf=uncloud.service
|
||||||
Requires=uncloud.service
|
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
|
|||||||
Reference in New Issue
Block a user