From 2134f3c1d98d2f49840b9847a12427e1fa087e1f Mon Sep 17 00:00:00 2001 From: Pavel Sviderski Date: Wed, 11 Dec 2024 12:17:15 +1000 Subject: [PATCH] add a stub for Caddyfile controller --- internal/machine/caddyfile/controller.go | 35 ++++++++++++++++++++++++ internal/machine/machine.go | 18 +++++++++++- internal/machine/network.go | 32 +++++++++++++++++----- internal/machine/store/store.go | 2 +- 4 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 internal/machine/caddyfile/controller.go diff --git a/internal/machine/caddyfile/controller.go b/internal/machine/caddyfile/controller.go new file mode 100644 index 00000000..d7208755 --- /dev/null +++ b/internal/machine/caddyfile/controller.go @@ -0,0 +1,35 @@ +package caddyfile + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "uncloud/internal/machine/store" +) + +// Controller monitors container changes in the cluster store and generates a configuration file for Caddy reverse +// proxy. The generated Caddyfile allows Caddy to route external traffic to service containers across the internal +// network. +type Controller struct { + store *store.Store + path string + mu sync.Mutex +} + +func NewController(store *store.Store, path string) (*Controller, error) { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("create parent directory for Caddyfile '%s': %w", dir, err) + } + + return &Controller{ + store: store, + path: path, + }, nil +} + +func (cc *Controller) Run(ctx context.Context) error { + return nil +} diff --git a/internal/machine/machine.go b/internal/machine/machine.go index 663d6ef4..3ed29ab5 100644 --- a/internal/machine/machine.go +++ b/internal/machine/machine.go @@ -25,6 +25,7 @@ import ( "uncloud/internal/fs" "uncloud/internal/machine/api/pb" apiproxy "uncloud/internal/machine/api/proxy" + "uncloud/internal/machine/caddyfile" "uncloud/internal/machine/cluster" "uncloud/internal/machine/corroservice" machinedocker "uncloud/internal/machine/docker" @@ -39,7 +40,7 @@ const ( ) type Config struct { - // DataDir is the directory where the machine stores its persistent state. + // DataDir is the directory where the machine stores its persistent state. Default is /var/lib/uncloud. DataDir string MachineSockPath string UncloudSockPath string @@ -54,6 +55,10 @@ type Config struct { // DockerClient manages system and user containers using the local Docker daemon. DockerClient *client.Client + + // CaddyfilePath specifies where the machine generates the Caddy reverse proxy configuration file for routing + // external traffic to service containers across the internal network. Default is DataDir/caddy/Caddyfile. + CaddyfilePath string } // SetDefaults returns a new Config with default values set where not provided. @@ -115,6 +120,11 @@ func (c *Config) SetDefaults() (*Config, error) { cfg.CorrosionService = corroservice.DefaultSystemdService(cfg.CorrosionDir) } } + + if cfg.CaddyfilePath == "" { + cfg.CaddyfilePath = filepath.Join(cfg.DataDir, "caddy", "Caddyfile") + } + return &cfg, nil } @@ -347,12 +357,18 @@ func (m *Machine) Run(ctx context.Context) error { ), ) + caddyfileCtrl, err := caddyfile.NewController(m.store, m.config.CaddyfilePath) + if err != nil { + return fmt.Errorf("create Caddyfile controller: %w", err) + } + ctrl, err = newNetworkController( m.state, m.store, proxyServer, m.config.CorrosionService, m.config.DockerClient, + caddyfileCtrl, ) if err != nil { return fmt.Errorf("initialise network controller: %w", err) diff --git a/internal/machine/network.go b/internal/machine/network.go index f41c1c16..859c9343 100644 --- a/internal/machine/network.go +++ b/internal/machine/network.go @@ -15,6 +15,7 @@ import ( "strconv" "time" "uncloud/internal/machine/api/pb" + "uncloud/internal/machine/caddyfile" "uncloud/internal/machine/corroservice" "uncloud/internal/machine/docker" "uncloud/internal/machine/network" @@ -32,16 +33,22 @@ type networkController struct { wgnet *network.WireGuardNetwork endpointChanges <-chan network.EndpointChangeEvent - server *grpc.Server - corroService corroservice.Service - dockerCli *client.Client + server *grpc.Server + corroService corroservice.Service + dockerCli *client.Client + caddyfileCtrl *caddyfile.Controller // TODO: DNS server/resolver listening on the machine IP, e.g. 10.210.0.1:53. It can't listen on 127.0.X.X // like resolved does because it needs to be reachable from both the host and the containers. } func newNetworkController( - state *State, store *store.Store, server *grpc.Server, corroService corroservice.Service, dockerCli *client.Client, + state *State, + store *store.Store, + server *grpc.Server, + corroService corroservice.Service, + dockerCli *client.Client, + caddyfileCtrl *caddyfile.Controller, ) ( *networkController, error, ) { @@ -60,6 +67,7 @@ func newNetworkController( server: server, corroService: corroService, dockerCli: dockerCli, + caddyfileCtrl: caddyfileCtrl, }, nil } @@ -139,7 +147,7 @@ func (nc *networkController) Run(ctx context.Context) error { break } } - if err = nc.state.Save(); err != nil { + if err := nc.state.Save(); err != nil { slog.Error("Failed to save machine state.", "err", err) } nc.state.mu.Unlock() @@ -155,13 +163,22 @@ func (nc *networkController) Run(ctx context.Context) error { errGroup.Go( func() error { - if err = nc.wgnet.Run(ctx); err != nil { + if err := nc.wgnet.Run(ctx); err != nil { return fmt.Errorf("WireGuard network failed: %w", err) } return nil }, ) + errGroup.Go(func() error { + slog.Info("Starting Caddyfile controller.") + if err := nc.caddyfileCtrl.Run(ctx); err != nil { + //goland:noinspection GoErrorStringFormat + return fmt.Errorf("Caddyfile controller failed: %w", err) + } + return nil + }) + // Wait for the context to be done and stop the network API server. errGroup.Go( func() error { @@ -214,7 +231,8 @@ func (nc *networkController) prepareAndWatchDocker(ctx context.Context) error { return nil } -// handleMachineChanges subscribes to machine changes in the cluster and reconfigures the network peers accordingly. +// handleMachineChanges subscribes to machine changes in the cluster and reconfigures the network peers accordingly +// when changes occur. func (nc *networkController) handleMachineChanges(ctx context.Context) error { for { // Retry to subscribe to machine changes indefinitely until the context is done. diff --git a/internal/machine/store/store.go b/internal/machine/store/store.go index 3fb0669a..6b746829 100644 --- a/internal/machine/store/store.go +++ b/internal/machine/store/store.go @@ -88,7 +88,7 @@ func (s *Store) ListMachines(ctx context.Context) ([]*pb.MachineInfo, error) { } // SubscribeMachines returns a list of machines and a channel that signals changes to the list. The channel doesn't -// receive any values, it's just signals when a machine has been added, updated, or deleted in the database. +// receive any values, it just signals when a machine has been added, updated, or deleted in the database. func (s *Store) SubscribeMachines(ctx context.Context) ([]*pb.MachineInfo, <-chan struct{}, error) { sub, err := s.corro.SubscribeContext(ctx, "SELECT info FROM machines ORDER BY name", nil, false) if err != nil {