From 8d16da596c6d09b0e6a664c9960870e69a50f83f Mon Sep 17 00:00:00 2001 From: Pavel Sviderski Date: Wed, 30 Apr 2025 17:24:52 +1000 Subject: [PATCH] feat(dns-server): run embedded internal DNS server on machine IP in uncloudd daemon process --- internal/machine/machine.go | 20 ++++ internal/machine/network.go | 143 ++++++++++++++++------------- internal/machine/network/config.go | 9 +- internal/machine/network/ip.go | 3 +- 4 files changed, 105 insertions(+), 70 deletions(-) diff --git a/internal/machine/machine.go b/internal/machine/machine.go index 15cf5b2e..f2531e17 100644 --- a/internal/machine/machine.go +++ b/internal/machine/machine.go @@ -24,6 +24,7 @@ import ( "github.com/psviderski/uncloud/internal/machine/caddyfile" "github.com/psviderski/uncloud/internal/machine/cluster" "github.com/psviderski/uncloud/internal/machine/corroservice" + "github.com/psviderski/uncloud/internal/machine/dns" machinedocker "github.com/psviderski/uncloud/internal/machine/docker" "github.com/psviderski/uncloud/internal/machine/network" "github.com/psviderski/uncloud/internal/machine/store" @@ -62,6 +63,8 @@ type Config struct { // CaddyConfigPath specifies where the machine generates the Caddy reverse proxy configuration file for routing // external traffic to service containers across the internal network. Default is DataDir/caddy/caddy.json. CaddyConfigPath string + // DNSUpstreams specifies the upstream DNS servers for the embedded internal DNS server. + DNSUpstreams []netip.AddrPort } // SetDefaults returns a new Config with default values set where not provided. @@ -270,6 +273,15 @@ func (m *Machine) Initialised() bool { return m.state.ID != "" } +// IP returns the machine IPv4 address in the cluster network which is the first address in the machine subnet. +func (m *Machine) IP() netip.Addr { + if !m.Initialised() { + return netip.Addr{} + } + + return network.MachineIP(m.state.Network.Subnet) +} + func (m *Machine) Run(ctx context.Context) error { // Docker dependency is essential for the machine to function. Block until it's ready. if err := docker.WaitDaemonReady(ctx, m.config.DockerClient); err != nil { @@ -373,6 +385,12 @@ func (m *Machine) Run(ctx context.Context) error { return fmt.Errorf("create Caddyfile controller: %w", err) } + dnsResolver := dns.NewClusterResolver(m.store) + dnsServer, err := dns.NewServer(m.IP(), dnsResolver, m.config.DNSUpstreams) + if err != nil { + return fmt.Errorf("create embedded DNS server: %w", err) + } + ctrl, err = newNetworkController( m.state, m.store, @@ -380,6 +398,8 @@ func (m *Machine) Run(ctx context.Context) error { m.config.CorrosionService, m.config.DockerClient, caddyfileCtrl, + dnsServer, + dnsResolver, ) if err != nil { return fmt.Errorf("initialise network controller: %w", err) diff --git a/internal/machine/network.go b/internal/machine/network.go index 3b384632..3d702c5d 100644 --- a/internal/machine/network.go +++ b/internal/machine/network.go @@ -4,22 +4,24 @@ import ( "context" "errors" "fmt" - "github.com/cenkalti/backoff/v4" - "github.com/docker/docker/client" - "golang.org/x/sync/errgroup" - "google.golang.org/grpc" "log/slog" "net" "net/netip" "slices" "strconv" "time" + + "github.com/cenkalti/backoff/v4" + "github.com/docker/docker/client" "github.com/psviderski/uncloud/internal/machine/api/pb" "github.com/psviderski/uncloud/internal/machine/caddyfile" "github.com/psviderski/uncloud/internal/machine/corroservice" + "github.com/psviderski/uncloud/internal/machine/dns" "github.com/psviderski/uncloud/internal/machine/docker" "github.com/psviderski/uncloud/internal/machine/network" "github.com/psviderski/uncloud/internal/machine/store" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" ) const ( @@ -38,8 +40,9 @@ type networkController struct { dockerCli *client.Client caddyfileCtrl *caddyfile.Controller - // TODO: DNS server/resolver listening on the machine IP, e.g. 10.210.0.1:53. It can't listen on 127.0.X.X - // like resolved does because it needs to be reachable from both the host and the containers. + // dnsServer is the embedded internal DNS server for the cluster listening on the machine IP. + dnsServer *dns.Server + dnsResolver *dns.ClusterResolver } func newNetworkController( @@ -49,6 +52,8 @@ func newNetworkController( corroService corroservice.Service, dockerCli *client.Client, caddyfileCtrl *caddyfile.Controller, + dnsServer *dns.Server, + dnsResolver *dns.ClusterResolver, ) ( *networkController, error, ) { @@ -68,6 +73,8 @@ func newNetworkController( corroService: corroService, dockerCli: dockerCli, caddyfileCtrl: caddyfileCtrl, + dnsServer: dnsServer, + dnsResolver: dnsResolver, }, nil } @@ -110,65 +117,73 @@ func (nc *networkController) Run(ctx context.Context) error { }, ) + errGroup.Go(func() error { + slog.Info("Starting embedded DNS resolver.") + if err := nc.dnsResolver.Run(ctx); err != nil { + return fmt.Errorf("embedded DNS resolver failed: %w", err) + } + return nil + }) + + errGroup.Go(func() error { + slog.Info("Starting embedded DNS server.") + if err := nc.dnsServer.Run(ctx); err != nil { + return fmt.Errorf("embedded DNS server failed: %w", err) + } + return nil + }) + // Setup Docker network and synchronise containers to the cluster store. - errGroup.Go( - func() error { - return nc.prepareAndWatchDocker(ctx) - }, - ) + errGroup.Go(func() error { + return nc.prepareAndWatchDocker(ctx) + }) // Handle machine changes in the cluster. Handling machine and endpoint changes should be done // in separate goroutines to avoid a deadlock when reconfiguring the network. - errGroup.Go( - func() error { - if err := nc.handleMachineChanges(ctx); err != nil { - return fmt.Errorf("handle new machines: %w", err) - } - return nil - }, - ) + errGroup.Go(func() error { + if err := nc.handleMachineChanges(ctx); err != nil { + return fmt.Errorf("handle new machines: %w", err) + } + return nil + }) // Watch for endpoint changes and update the machine state accordingly. - errGroup.Go( - func() error { - for { - select { - case e, ok := <-nc.endpointChanges: - if !ok { - // The channel was closed, stop watching for changes. - nc.endpointChanges = nil - return nil - } - - nc.state.mu.Lock() - for i := range nc.state.Network.Peers { - if nc.state.Network.Peers[i].PublicKey.Equal(e.PublicKey) { - nc.state.Network.Peers[i].Endpoint = &e.Endpoint - break - } - } - if err := nc.state.Save(); err != nil { - slog.Error("Failed to save machine state.", "err", err) - } - nc.state.mu.Unlock() - - slog.Debug("Preserved endpoint change in the machine state.", - "public_key", e.PublicKey, "endpoint", e.Endpoint) - case <-ctx.Done(): + errGroup.Go(func() error { + for { + select { + case e, ok := <-nc.endpointChanges: + if !ok { + // The channel was closed, stop watching for changes. + nc.endpointChanges = nil return nil } - } - }, - ) - errGroup.Go( - func() error { - if err := nc.wgnet.Run(ctx); err != nil { - return fmt.Errorf("WireGuard network failed: %w", err) + nc.state.mu.Lock() + for i := range nc.state.Network.Peers { + if nc.state.Network.Peers[i].PublicKey.Equal(e.PublicKey) { + nc.state.Network.Peers[i].Endpoint = &e.Endpoint + break + } + } + if err := nc.state.Save(); err != nil { + slog.Error("Failed to save machine state.", "err", err) + } + nc.state.mu.Unlock() + + slog.Debug("Preserved endpoint change in the machine state.", + "public_key", e.PublicKey, "endpoint", e.Endpoint) + case <-ctx.Done(): + return nil } - return nil - }, - ) + } + }) + + errGroup.Go(func() error { + if err := nc.wgnet.Run(ctx); err != nil { + return fmt.Errorf("WireGuard network failed: %w", err) + } + return nil + }) errGroup.Go(func() error { slog.Info("Starting Caddyfile controller.") @@ -180,16 +195,14 @@ func (nc *networkController) Run(ctx context.Context) error { }) // Wait for the context to be done and stop the network API server. - errGroup.Go( - func() error { - <-ctx.Done() - slog.Info("Stopping network API server.") - // TODO: implement timeout for graceful shutdown. - nc.server.GracefulStop() - slog.Info("Network API server stopped.") - return nil - }, - ) + errGroup.Go(func() error { + <-ctx.Done() + slog.Info("Stopping network API server.") + // TODO: implement timeout for graceful shutdown. + nc.server.GracefulStop() + slog.Info("Network API server stopped.") + return nil + }) return errGroup.Wait() } diff --git a/internal/machine/network/config.go b/internal/machine/network/config.go index 3de8d45a..81b0b145 100644 --- a/internal/machine/network/config.go +++ b/internal/machine/network/config.go @@ -2,10 +2,11 @@ package network import ( "fmt" - "github.com/psviderski/uncloud/internal/secret" - "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "net" "net/netip" + + "github.com/psviderski/uncloud/internal/secret" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) type Config struct { @@ -13,7 +14,7 @@ type Config struct { // in the subnet. Other IP addresses are allocated to containers running on the machine. Subnet netip.Prefix // ManagementIP is the IPv6 address assigned to the machine within the WireGuard network. This address is used - // for cluster management traffic, such as gRPC communication with the machine API server and Serf gossip. + // for cluster management traffic, such as gRPC communication with the machine API server and Corrosion gossip. ManagementIP netip.Addr PrivateKey secret.Secret PublicKey secret.Secret @@ -23,7 +24,7 @@ type Config struct { type PeerConfig struct { Subnet *netip.Prefix `json:",omitempty"` // ManagementIP is the IPv6 address assigned to the peer within the WireGuard network. This address is used - // for cluster management traffic, such as gRPC communication with the machine API server and Serf gossip. + // for cluster management traffic, such as gRPC communication with the machine API server and Corrosion gossip. ManagementIP netip.Addr Endpoint *netip.AddrPort `json:",omitempty"` AllEndpoints []netip.AddrPort `json:",omitempty"` diff --git a/internal/machine/network/ip.go b/internal/machine/network/ip.go index 1a7f33c7..4b6a3833 100644 --- a/internal/machine/network/ip.go +++ b/internal/machine/network/ip.go @@ -2,9 +2,10 @@ package network import ( "fmt" - "github.com/psviderski/uncloud/internal/secret" "net" "net/netip" + + "github.com/psviderski/uncloud/internal/secret" ) // MachineIP returns the IP address of the machine which is the first address in the subnet.