From b39585df4cf15e20da96e99e4572df4aa8f62418 Mon Sep 17 00:00:00 2001 From: Pasha Sviderski Date: Mon, 4 Aug 2025 19:09:38 +1000 Subject: [PATCH] chore: do not masquerade container IPs when communicating via WG mesh --- internal/machine/cluster.go | 23 +++++--- internal/machine/docker/manager_linux.go | 61 ++++++++++++++++++--- internal/machine/network/wireguard_linux.go | 19 ++++--- 3 files changed, 79 insertions(+), 24 deletions(-) diff --git a/internal/machine/cluster.go b/internal/machine/cluster.go index 7a547177..70b03b50 100644 --- a/internal/machine/cluster.go +++ b/internal/machine/cluster.go @@ -94,6 +94,11 @@ func (cc *clusterController) Run(ctx context.Context) error { return fmt.Errorf("configure iptables chains: %w", err) } + if err := cc.ensureDockerNetwork(ctx); err != nil { + return err + } + slog.Info("Docker network configured.") + if err := cc.wgnet.Configure(*cc.state.Network); err != nil { return fmt.Errorf("configure WireGuard network: %w", err) } @@ -138,6 +143,7 @@ func (cc *clusterController) Run(ctx context.Context) error { return nil }) + // The Docker network must be created before starting the DNS server because it listens on the machine IP. errGroup.Go(func() error { slog.Info("Starting embedded DNS server.") if err := cc.dnsServer.Run(ctx); err != nil { @@ -146,9 +152,10 @@ func (cc *clusterController) Run(ctx context.Context) error { return nil }) - // Setup Docker network and synchronise containers to the cluster store. + // Synchronise Docker containers to the cluster store. errGroup.Go(func() error { - return cc.prepareAndWatchDocker(ctx) + slog.Info("Watching Docker containers and syncing them to cluster store.") + return cc.syncDockerContainers(ctx) }) // Handle machine changes in the cluster. Handling machine and endpoint changes should be done @@ -229,9 +236,8 @@ func (cc *clusterController) Run(ctx context.Context) error { return err } -// prepareAndWatchDocker configures the Docker network and watches local Docker containers to sync them -// to the cluster store. -func (cc *clusterController) prepareAndWatchDocker(ctx context.Context) error { +// ensureDockerNetwork ensures that the Docker network is configured and ready for containers. +func (cc *clusterController) ensureDockerNetwork(ctx context.Context) error { if err := cc.dockerManager.WaitDaemonReady(ctx); err != nil { return fmt.Errorf("wait for Docker daemon: %w", err) } @@ -243,12 +249,15 @@ func (cc *clusterController) prepareAndWatchDocker(ctx context.Context) error { ); err != nil { return fmt.Errorf("ensure Docker network: %w", err) } - slog.Info("Docker network configured.") // Signal that Docker is ready for containers. close(cc.dockerReady) - slog.Info("Watching Docker containers and syncing them to cluster store.") + return nil +} + +// syncDockerContainers watches local Docker containers and syncs them to the cluster store. +func (cc *clusterController) syncDockerContainers(ctx context.Context) error { // Retry to watch and sync containers until the context is done. boff := backoff.WithContext(backoff.NewExponentialBackOff( backoff.WithInitialInterval(100*time.Millisecond), diff --git a/internal/machine/docker/manager_linux.go b/internal/machine/docker/manager_linux.go index 08c20d7f..98c5272f 100644 --- a/internal/machine/docker/manager_linux.go +++ b/internal/machine/docker/manager_linux.go @@ -84,7 +84,7 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, // https://github.com/moby/moby/blob/v27.2.1/libnetwork/drivers/bridge/bridge_linux.go#L664 bridgeName := "br-" + nw.ID[:12] - if err = configureIptables(bridgeName, dnsServer); err != nil { + if err = configureIptables(bridgeName, subnet, dnsServer); err != nil { return fmt.Errorf("configure iptables for Docker network '%s': %w", NetworkName, err) } @@ -92,11 +92,15 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, } // configureIptables configures iptables rules for the uncloud Docker network. -func configureIptables(bridgeName string, dnsServer netip.Addr) error { +func configureIptables(bridgeName string, subnet netip.Prefix, dnsServer netip.Addr) error { ipt := iptables.GetIptable(iptables.IPv4) // Allow traffic from other machines and their containers through the WG mesh to the Uncloud containers // on the machine. - wgRule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"} + wgRule := []string{ + "--in-interface", network.WireGuardInterfaceName, + "--out-interface", bridgeName, + "-j", "ACCEPT", + } if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Insert, wgRule); err != nil { return fmt.Errorf("insert iptables rule: %w", err) } @@ -115,17 +119,49 @@ func configureIptables(bridgeName string, dnsServer netip.Addr) error { } } + // Skip masquerading for the container traffic going from the uncloud Docker network through the WG mesh. + // https://uncloud.run/blog/connect-docker-containers-across-hosts-wireguard#step-3-configure-ip-routing + skipMasqueradeRule := []string{ + "--src", subnet.String(), + "--out-interface", network.WireGuardInterfaceName, + "-j", "RETURN", + } + // Delete and reinsert the rule to ensure it's at the top of the POSTROUTING chain before the MASQUERADE rule + // added by Docker: POSTROUTING -s 10.210.X.0/24 ! -o br-XXX -j MASQUERADE + if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil { + return fmt.Errorf("delete iptables rule: %w", err) + } + if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Insert, skipMasqueradeRule); err != nil { + return fmt.Errorf("insert iptables rule: %w", err) + } + return nil } // cleanupIptables deletes the iptables rules for the uncloud Docker network. -func cleanupIptables(bridgeName string) error { +func cleanupIptables(bridgeName string, subnet netip.Prefix) error { ipt := iptables.GetIptable(iptables.IPv4) // Delete the rule allowing traffic from the WireGuard network to the Docker bridge. - wgRule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"} + wgRule := []string{ + "--in-interface", network.WireGuardInterfaceName, + "--out-interface", bridgeName, + "-j", "ACCEPT", + } if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Delete, wgRule); err != nil { return fmt.Errorf("delete iptables rule: %w", err) } + + // Delete the rule that skips masquerading for the container traffic going from the uncloud Docker network + // through the WG mesh. + skipMasqueradeRule := []string{ + "--src", subnet.String(), + "--out-interface", network.WireGuardInterfaceName, + "-j", "RETURN", + } + if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil { + return fmt.Errorf("delete iptables rule: %w", err) + } + // Rules in uncloud-owned chains will be automatically cleaned up by the machine cleanup. return nil @@ -172,10 +208,17 @@ func (m *Manager) Cleanup() error { nw, err := m.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{}) if err == nil { bridgeName := "br-" + nw.ID[:12] - if err = cleanupIptables(bridgeName); err != nil { - errs = append(errs, fmt.Errorf("cleanup iptables for Docker network '%s': %w", NetworkName, err)) - } else { - slog.Info("Cleaned up iptables rules for Docker network.", "name", NetworkName, "bridge", bridgeName) + var subnet netip.Prefix + if len(nw.IPAM.Config) > 0 { + subnet, _ = netip.ParsePrefix(nw.IPAM.Config[0].Subnet) + } + + if subnet.IsValid() { + if err = cleanupIptables(bridgeName, subnet); err != nil { + errs = append(errs, fmt.Errorf("cleanup iptables for Docker network '%s': %w", NetworkName, err)) + } else { + slog.Info("Cleaned up iptables rules for Docker network.", "name", NetworkName, "bridge", bridgeName) + } } if err = m.client.NetworkRemove(ctx, NetworkName); err == nil { diff --git a/internal/machine/network/wireguard_linux.go b/internal/machine/network/wireguard_linux.go index 111237fc..75fe6107 100644 --- a/internal/machine/network/wireguard_linux.go +++ b/internal/machine/network/wireguard_linux.go @@ -81,19 +81,15 @@ func (n *WireGuardNetwork) Configure(config Config) error { } slog.Info("Configured WireGuard interface.", "name", n.link.Attrs().Name) - machinePrefix := netip.PrefixFrom(MachineIP(config.Subnet), config.Subnet.Bits()) managementPrefix, err := addrToSingleIPPrefix(config.ManagementIP) if err != nil { return fmt.Errorf("parse management IP: %w", err) } - addrs := []netip.Prefix{managementPrefix, machinePrefix} + addrs := []netip.Prefix{managementPrefix} if err = n.updateAddresses(addrs); err != nil { return err } - slog.Info( - "Updated addresses of the WireGuard interface.", - "name", n.link.Attrs().Name, "addrs", addrs, - ) + slog.Info("Updated addresses of the WireGuard interface.", "name", n.link.Attrs().Name, "addrs", addrs) // Bring the WireGuard interface up if it's not already up. if n.link.Attrs().Flags&unix.IFF_UP != unix.IFF_UP { @@ -102,7 +98,7 @@ func (n *WireGuardNetwork) Configure(config Config) error { } slog.Info("Brought WireGuard interface up.", "name", n.link.Attrs().Name) } - if err = n.updatePeerRoutes(); err != nil { + if err = n.updatePeerRoutes(MachineIP(config.Subnet)); err != nil { return err } slog.Info( @@ -205,7 +201,7 @@ func (n *WireGuardNetwork) updateAddresses(addrs []netip.Prefix) error { // updatePeerRoutes adds routes to the peers via the WireGuard interface and removes old routes to peers // that are no longer in the configuration. -func (n *WireGuardNetwork) updatePeerRoutes() error { +func (n *WireGuardNetwork) updatePeerRoutes(machineIP netip.Addr) error { // Build a set of compacted IP ranges for all peers. var ipsetBuilder netipx.IPSetBuilder for _, p := range n.peers { @@ -225,11 +221,18 @@ func (n *WireGuardNetwork) updatePeerRoutes() error { // Add routes to the computed IP ranges via the WireGuard link. for _, prefix := range ipset.Prefixes() { dst := prefixToIPNet(prefix) + var src net.IP + // Use the machine IP as the source address for IPv4 routes to other peers. + if prefix.Addr().Is4() { + src = machineIP.AsSlice() + } + if err = netlink.RouteAdd( &netlink.Route{ LinkIndex: n.link.Attrs().Index, Scope: netlink.SCOPE_LINK, Dst: &dst, + Src: src, }, ); err != nil { if !errors.Is(err, unix.EEXIST) {