From 3648187219fd9ba3a382a06c914c9d2d164704a6 Mon Sep 17 00:00:00 2001 From: Pavel Sviderski Date: Mon, 5 May 2025 20:00:34 +1000 Subject: [PATCH] chore: allow DNS queries from Uncloud containers to the embedded DNS server --- internal/machine/docker/manager.go | 6 +- internal/machine/docker/manager_darwin.go | 6 +- internal/machine/docker/manager_linux.go | 43 ++++++++++-- internal/machine/firewall/iptables_darwin.go | 8 +++ internal/machine/firewall/iptables_linux.go | 74 ++++++++++++++++++++ internal/machine/network.go | 74 ++------------------ 6 files changed, 128 insertions(+), 83 deletions(-) create mode 100644 internal/machine/firewall/iptables_darwin.go create mode 100644 internal/machine/firewall/iptables_linux.go diff --git a/internal/machine/docker/manager.go b/internal/machine/docker/manager.go index 065ac5ac..19d9a7da 100644 --- a/internal/machine/docker/manager.go +++ b/internal/machine/docker/manager.go @@ -4,19 +4,19 @@ import ( "context" "errors" "fmt" + "log/slog" + "time" + dockercontainer "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/events" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" "github.com/psviderski/uncloud/internal/machine/store" "github.com/psviderski/uncloud/pkg/api" - "log/slog" - "time" ) const ( NetworkName = "uncloud" - UserChain = "DOCKER-USER" // EventsDebounceInterval defines how long to wait before processing the next Docker event. Multiple events // occurring within this window will be processed together as a single event to prevent system overload. EventsDebounceInterval = 100 * time.Millisecond diff --git a/internal/machine/docker/manager_darwin.go b/internal/machine/docker/manager_darwin.go index cc11192a..7f65b1c7 100644 --- a/internal/machine/docker/manager_darwin.go +++ b/internal/machine/docker/manager_darwin.go @@ -8,7 +8,7 @@ import ( "net/netip" ) -// EnsureUncloudNetwork is a stub for darwin. -func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix) error { - return fmt.Errorf("not supported on darwin") +// EnsureUncloudNetwork is a stub for Darwin. +func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error { + return fmt.Errorf("not supported on Darwin") } diff --git a/internal/machine/docker/manager_linux.go b/internal/machine/docker/manager_linux.go index d736da35..cbd7b067 100644 --- a/internal/machine/docker/manager_linux.go +++ b/internal/machine/docker/manager_linux.go @@ -5,23 +5,26 @@ import ( "fmt" "log/slog" "net/netip" + "strconv" dnetwork "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/docker/libnetwork/iptables" + "github.com/psviderski/uncloud/internal/machine/dns" + "github.com/psviderski/uncloud/internal/machine/firewall" "github.com/psviderski/uncloud/internal/machine/network" ) // EnsureUncloudNetwork creates the Docker bridge network NetworkName with the provided machine subnet // if it doesn't exist. If the network exists but has a different subnet, it removes and recreates the network. // It also configures iptables to allow container access from the WireGuard network. -func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix) error { +func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error { // Ensure the Docker network 'uncloud' is created with the correct subnet. needsCreation := false nw, err := m.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{}) if err != nil { if !client.IsErrNotFound(err) { - return fmt.Errorf("inspect Docker network %q: %w", NetworkName, err) + return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err) } needsCreation = true } else if nw.IPAM.Config[0].Subnet != subnet.String() { @@ -32,7 +35,7 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix) ) if err = m.client.NetworkRemove(ctx, NetworkName); err != nil { // It can still fail if the network is in use by a container. Leave it to the user to resolve the issue. - return fmt.Errorf("remove Docker network %q: %w", NetworkName, err) + return fmt.Errorf("remove Docker network '%s': %w", NetworkName, err) } needsCreation = true } @@ -51,12 +54,12 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix) }, }, ); err != nil { - return fmt.Errorf("create Docker network %q: %w", NetworkName, err) + return fmt.Errorf("create Docker network '%s': %w", NetworkName, err) } slog.Info("Docker network created.", "name", NetworkName, "subnet", subnet.String()) if nw, err = m.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{}); err != nil { - return fmt.Errorf("inspect Docker network %q: %w", NetworkName, err) + return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err) } } @@ -68,10 +71,36 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix) // Bridge name doesn't seem to be documented but this is the source code where it is generated: // https://github.com/moby/moby/blob/v27.2.1/libnetwork/drivers/bridge/bridge_linux.go#L664 bridgeName := "br-" + nw.ID[:12] + + if err = configureIptables(bridgeName, dnsServer); err != nil { + return fmt.Errorf("configure iptables for Docker network '%s': %w", NetworkName, err) + } + + return nil +} + +// configureIptables configures iptables rules for the uncloud Docker network. +func configureIptables(bridgeName string, dnsServer netip.Addr) error { ipt := iptables.GetIptable(iptables.IPv4) - rule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"} - if err = ipt.ProgramRule(iptables.Filter, UserChain, iptables.Insert, rule); err != nil { + // Allow traffic from other machines and their containers through the WG mesh to the Uncloud containers + // on the machine. + wgRule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"} + if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Insert, wgRule); err != nil { return fmt.Errorf("insert iptables rule: %w", err) + } + + // Allow DNS queries from Uncloud containers to the embedded DNS server. + for _, proto := range []string{"udp", "tcp"} { + dnsRule := []string{ + "--in-interface", bridgeName, + "--dst", dnsServer.String(), + "--protocol", proto, + "--dport", strconv.Itoa(dns.Port), + "-j", "ACCEPT", + } + if err := ipt.ProgramRule(iptables.Filter, firewall.UncloudInputChain, iptables.Insert, dnsRule); err != nil { + return fmt.Errorf("insert iptables rule: %w", err) + } } return nil diff --git a/internal/machine/firewall/iptables_darwin.go b/internal/machine/firewall/iptables_darwin.go new file mode 100644 index 00000000..5202f531 --- /dev/null +++ b/internal/machine/firewall/iptables_darwin.go @@ -0,0 +1,8 @@ +package firewall + +import "fmt" + +// ConfigureIptablesChains is a stub for Darwin. +func ConfigureIptablesChains() error { + return fmt.Errorf("not supported on Darwin") +} diff --git a/internal/machine/firewall/iptables_linux.go b/internal/machine/firewall/iptables_linux.go new file mode 100644 index 00000000..2ede6805 --- /dev/null +++ b/internal/machine/firewall/iptables_linux.go @@ -0,0 +1,74 @@ +package firewall + +import ( + "fmt" + "strconv" + "strings" + + "github.com/docker/docker/libnetwork/iptables" + "github.com/psviderski/uncloud/internal/machine/network" +) + +const ( + DockerUserChain = "DOCKER-USER" + UncloudInputChain = "UNCLOUD-INPUT" +) + +// ConfigureIptablesChains sets up custom iptables chains and initial firewall rules for Uncloud networking. +func ConfigureIptablesChains() error { + // Ensure iptables UNCLOUD-INPUT chain with a RETURN rule exists. All existing rules are flushed. + ipt := iptables.GetIptable(iptables.IPv4) + if _, err := ipt.NewChain(UncloudInputChain, iptables.Filter); err != nil { + return fmt.Errorf("create iptables chain '%s': %w", UncloudInputChain, err) + } + if err := ipt.RawCombinedOutput("-t", string(iptables.Filter), "-F", UncloudInputChain); err != nil { + return fmt.Errorf("flush iptables chain '%s': %w", UncloudInputChain, err) + } + if err := ipt.AddReturnRule(UncloudInputChain); err != nil { + return fmt.Errorf("add the RETURN rule for iptables chain '%s': %w", UncloudInputChain, err) + } + + // Ensure the main iptables INPUT chain has a jump rule to the UNCLOUD-INPUT chain before any DROP/REJECT rules. + jumpRule := []string{"-m", "comment", "--comment", "Uncloud-managed", "-j", UncloudInputChain} + if !ipt.Exists(iptables.Filter, "INPUT", jumpRule...) { + // Look for the first DROP/REJECT rule in the INPUT chain. + out, err := ipt.Raw("-t", string(iptables.Filter), "-L", "INPUT", "--line-numbers") + if err != nil { + return fmt.Errorf("get iptables rules for chain '%s': %w", UncloudInputChain, err) + } + + firstRejectRuleNum := 0 + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + if fields[1] == "DROP" || fields[1] == "REJECT" { + if ruleNum, err := strconv.Atoi(fields[0]); err == nil { + firstRejectRuleNum = ruleNum + break + } + } + } + + var addJumpRule []string + if firstRejectRuleNum > 0 { + addJumpRule = append([]string{"-t", string(iptables.Filter), "-I", "INPUT", strconv.Itoa(firstRejectRuleNum)}, + jumpRule...) + } else { + addJumpRule = append([]string{"-t", string(iptables.Filter), "-A", "INPUT"}, jumpRule...) + } + if err = ipt.RawCombinedOutput(addJumpRule...); err != nil { + return fmt.Errorf("add iptables rule '%s': %w", strings.Join(addJumpRule, " "), err) + } + } + + // Allow WireGuard traffic to the machine. + acceptWireGuardRule := []string{"-p", "udp", "--dport", strconv.Itoa(network.WireGuardPort), "-j", "ACCEPT"} + err := ipt.ProgramRule(iptables.Filter, UncloudInputChain, iptables.Insert, acceptWireGuardRule) + if err != nil { + return fmt.Errorf("insert iptables rule '%s': %w", strings.Join(acceptWireGuardRule, " "), err) + } + + return nil +} diff --git a/internal/machine/network.go b/internal/machine/network.go index 2517b50e..5046ba35 100644 --- a/internal/machine/network.go +++ b/internal/machine/network.go @@ -9,17 +9,16 @@ import ( "net/netip" "slices" "strconv" - "strings" "time" "github.com/cenkalti/backoff/v4" "github.com/docker/docker/client" - "github.com/docker/docker/libnetwork/iptables" "github.com/psviderski/uncloud/internal/machine/api/pb" "github.com/psviderski/uncloud/internal/machine/caddyfile" "github.com/psviderski/uncloud/internal/machine/corroservice" "github.com/psviderski/uncloud/internal/machine/dns" "github.com/psviderski/uncloud/internal/machine/docker" + "github.com/psviderski/uncloud/internal/machine/firewall" "github.com/psviderski/uncloud/internal/machine/network" "github.com/psviderski/uncloud/internal/machine/store" "golang.org/x/sync/errgroup" @@ -27,8 +26,7 @@ import ( ) const ( - APIPort = 51000 - iptablesUncloudInputChain = "UNCLOUD-INPUT" + APIPort = 51000 ) type networkController struct { @@ -82,7 +80,7 @@ func newNetworkController( } func (nc *networkController) Run(ctx context.Context) error { - if err := nc.configureIptablesChains(); err != nil { + if err := firewall.ConfigureIptablesChains(); err != nil { return fmt.Errorf("configure iptables chains: %w", err) } @@ -214,65 +212,6 @@ func (nc *networkController) Run(ctx context.Context) error { return errGroup.Wait() } -// configureIptablesChains sets up custom iptables chains and initial firewall rules for Uncloud networking. -func (nc *networkController) configureIptablesChains() error { - // Ensure iptables UNCLOUD-INPUT chain with a RETURN rule exists. All existing rules are flushed. - ipt := iptables.GetIptable(iptables.IPv4) - if _, err := ipt.NewChain(iptablesUncloudInputChain, iptables.Filter); err != nil { - return fmt.Errorf("create iptables chain '%s': %w", iptablesUncloudInputChain, err) - } - if err := ipt.RawCombinedOutput("-t", string(iptables.Filter), "-F", iptablesUncloudInputChain); err != nil { - return fmt.Errorf("flush iptables chain '%s': %w", iptablesUncloudInputChain, err) - } - if err := ipt.AddReturnRule(iptablesUncloudInputChain); err != nil { - return fmt.Errorf("add the RETURN rule for iptables chain '%s': %w", iptablesUncloudInputChain, err) - } - - // Ensure the main iptables INPUT chain has a jump rule to the UNCLOUD-INPUT chain before any DROP/REJECT rules. - jumpRule := []string{"-m", "comment", "--comment", "Uncloud-managed", "-j", iptablesUncloudInputChain} - if !ipt.Exists(iptables.Filter, "INPUT", jumpRule...) { - // Look for the first DROP/REJECT rule in the INPUT chain. - out, err := ipt.Raw("-t", string(iptables.Filter), "-L", "INPUT", "--line-numbers") - if err != nil { - return fmt.Errorf("get iptables rules for chain '%s': %w", iptablesUncloudInputChain, err) - } - - firstRejectRuleNum := 0 - for _, line := range strings.Split(string(out), "\n") { - fields := strings.Fields(line) - if len(fields) < 2 { - continue - } - if fields[1] == "DROP" || fields[1] == "REJECT" { - if ruleNum, err := strconv.Atoi(fields[0]); err == nil { - firstRejectRuleNum = ruleNum - break - } - } - } - - var addJumpRule []string - if firstRejectRuleNum > 0 { - addJumpRule = append([]string{"-t", string(iptables.Filter), "-I", "INPUT", strconv.Itoa(firstRejectRuleNum)}, - jumpRule...) - } else { - addJumpRule = append([]string{"-t", string(iptables.Filter), "-A", "INPUT"}, jumpRule...) - } - if err = ipt.RawCombinedOutput(addJumpRule...); err != nil { - return fmt.Errorf("add iptables rule '%s': %w", strings.Join(addJumpRule, " "), err) - } - } - - // Allow WireGuard traffic to the machine. - acceptWireGuardRule := []string{"-p", "udp", "--dport", strconv.Itoa(network.WireGuardPort), "-j", "ACCEPT"} - err := ipt.ProgramRule(iptables.Filter, iptablesUncloudInputChain, iptables.Insert, acceptWireGuardRule) - if err != nil { - return fmt.Errorf("insert iptables rule '%s': %w", strings.Join(acceptWireGuardRule, " "), err) - } - - return nil -} - // prepareAndWatchDocker configures the Docker network and watches local Docker containers to sync them // to the cluster store. func (nc *networkController) prepareAndWatchDocker(ctx context.Context) error { @@ -281,16 +220,11 @@ func (nc *networkController) prepareAndWatchDocker(ctx context.Context) error { return fmt.Errorf("wait for Docker daemon: %w", err) } - if err := manager.EnsureUncloudNetwork(ctx, nc.state.Network.Subnet); err != nil { + if err := manager.EnsureUncloudNetwork(ctx, nc.state.Network.Subnet, nc.dnsServer.ListenAddr()); err != nil { return fmt.Errorf("ensure Docker network: %w", err) } slog.Info("Docker network configured.") - // TODO: add iptables rules to UNCLOUD-INPUT to allow DNS queries from Uncloud containers - // to the embedded DNS server: - // iptables -A UNCLOUD-INPUT -d 10.210.0.1/32 -i br-f6db1df6d60b -p tcp -m tcp --dport 53 -j ACCEPT - //. iptables -A UNCLOUD-INPUT -d 10.210.0.1/32 -i br-f6db1df6d60b -p udp -m udp --dport 53 -j ACCEPT - slog.Info("Watching Docker containers and syncing them to cluster store.") // Retry to watch and sync containers until the context is done. boff := backoff.WithContext(backoff.NewExponentialBackOff(