chore: do not masquerade container IPs when communicating via WG mesh

internal/machine/cluster.go

@@ -94,6 +94,11 @@ func (cc *clusterController) Run(ctx context.Context) error {
 		return fmt.Errorf("configure iptables chains: %w", err)
 	}
 
+	if err := cc.ensureDockerNetwork(ctx); err != nil {
+		return err
+	}
+	slog.Info("Docker network configured.")
+
 	if err := cc.wgnet.Configure(*cc.state.Network); err != nil {
 		return fmt.Errorf("configure WireGuard network: %w", err)
 	}
@@ -138,6 +143,7 @@ func (cc *clusterController) Run(ctx context.Context) error {
 		return nil
 	})
 
+	// The Docker network must be created before starting the DNS server because it listens on the machine IP.
 	errGroup.Go(func() error {
 		slog.Info("Starting embedded DNS server.")
 		if err := cc.dnsServer.Run(ctx); err != nil {
@@ -146,9 +152,10 @@ func (cc *clusterController) Run(ctx context.Context) error {
 		return nil
 	})
 
-	// Setup Docker network and synchronise containers to the cluster store.
+	// Synchronise Docker containers to the cluster store.
 	errGroup.Go(func() error {
-		return cc.prepareAndWatchDocker(ctx)
+		slog.Info("Watching Docker containers and syncing them to cluster store.")
+		return cc.syncDockerContainers(ctx)
 	})
 
 	// Handle machine changes in the cluster. Handling machine and endpoint changes should be done
@@ -229,9 +236,8 @@ func (cc *clusterController) Run(ctx context.Context) error {
 	return err
 }
 
-// prepareAndWatchDocker configures the Docker network and watches local Docker containers to sync them
+// ensureDockerNetwork ensures that the Docker network is configured and ready for containers.
-// to the cluster store.
+func (cc *clusterController) ensureDockerNetwork(ctx context.Context) error {
-func (cc *clusterController) prepareAndWatchDocker(ctx context.Context) error {
 	if err := cc.dockerManager.WaitDaemonReady(ctx); err != nil {
 		return fmt.Errorf("wait for Docker daemon: %w", err)
 	}
@@ -243,12 +249,15 @@ func (cc *clusterController) prepareAndWatchDocker(ctx context.Context) error {
 	); err != nil {
 		return fmt.Errorf("ensure Docker network: %w", err)
 	}
-	slog.Info("Docker network configured.")
 
 	// Signal that Docker is ready for containers.
 	close(cc.dockerReady)
 
-	slog.Info("Watching Docker containers and syncing them to cluster store.")
+	return nil
+}
+
+// syncDockerContainers watches local Docker containers and syncs them to the cluster store.
+func (cc *clusterController) syncDockerContainers(ctx context.Context) error {
 	// Retry to watch and sync containers until the context is done.
 	boff := backoff.WithContext(backoff.NewExponentialBackOff(
 		backoff.WithInitialInterval(100*time.Millisecond),

internal/machine/docker/manager_linux.go

@@ -84,7 +84,7 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix,
 	// https://github.com/moby/moby/blob/v27.2.1/libnetwork/drivers/bridge/bridge_linux.go#L664
 	bridgeName := "br-" + nw.ID[:12]
 
-	if err = configureIptables(bridgeName, dnsServer); err != nil {
+	if err = configureIptables(bridgeName, subnet, dnsServer); err != nil {
 		return fmt.Errorf("configure iptables for Docker network '%s': %w", NetworkName, err)
 	}
 
@@ -92,11 +92,15 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix,
 }
 
 // configureIptables configures iptables rules for the uncloud Docker network.
-func configureIptables(bridgeName string, dnsServer netip.Addr) error {
+func configureIptables(bridgeName string, subnet netip.Prefix, dnsServer netip.Addr) error {
 	ipt := iptables.GetIptable(iptables.IPv4)
 	// Allow traffic from other machines and their containers through the WG mesh to the Uncloud containers
 	// on the machine.
-	wgRule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"}
+	wgRule := []string{
+		"--in-interface", network.WireGuardInterfaceName,
+		"--out-interface", bridgeName,
+		"-j", "ACCEPT",
+	}
 	if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Insert, wgRule); err != nil {
 		return fmt.Errorf("insert iptables rule: %w", err)
 	}
@@ -115,17 +119,49 @@ func configureIptables(bridgeName string, dnsServer netip.Addr) error {
 		}
 	}
 
+	// Skip masquerading for the container traffic going from the uncloud Docker network through the WG mesh.
+	// https://uncloud.run/blog/connect-docker-containers-across-hosts-wireguard#step-3-configure-ip-routing
+	skipMasqueradeRule := []string{
+		"--src", subnet.String(),
+		"--out-interface", network.WireGuardInterfaceName,
+		"-j", "RETURN",
+	}
+	// Delete and reinsert the rule to ensure it's at the top of the POSTROUTING chain before the MASQUERADE rule
+	// added by Docker: POSTROUTING -s 10.210.X.0/24 ! -o br-XXX -j MASQUERADE
+	if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil {
+		return fmt.Errorf("delete iptables rule: %w", err)
+	}
+	if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Insert, skipMasqueradeRule); err != nil {
+		return fmt.Errorf("insert iptables rule: %w", err)
+	}
+
 	return nil
 }
 
 // cleanupIptables deletes the iptables rules for the uncloud Docker network.
-func cleanupIptables(bridgeName string) error {
+func cleanupIptables(bridgeName string, subnet netip.Prefix) error {
 	ipt := iptables.GetIptable(iptables.IPv4)
 	// Delete the rule allowing traffic from the WireGuard network to the Docker bridge.
-	wgRule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"}
+	wgRule := []string{
+		"--in-interface", network.WireGuardInterfaceName,
+		"--out-interface", bridgeName,
+		"-j", "ACCEPT",
+	}
 	if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Delete, wgRule); err != nil {
 		return fmt.Errorf("delete iptables rule: %w", err)
 	}
+
+	// Delete the rule that skips masquerading for the container traffic going from the uncloud Docker network
+	// through the WG mesh.
+	skipMasqueradeRule := []string{
+		"--src", subnet.String(),
+		"--out-interface", network.WireGuardInterfaceName,
+		"-j", "RETURN",
+	}
+	if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil {
+		return fmt.Errorf("delete iptables rule: %w", err)
+	}
+
 	// Rules in uncloud-owned chains will be automatically cleaned up by the machine cleanup.
 
 	return nil
@@ -172,11 +208,18 @@ func (m *Manager) Cleanup() error {
 	nw, err := m.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{})
 	if err == nil {
 		bridgeName := "br-" + nw.ID[:12]
-		if err = cleanupIptables(bridgeName); err != nil {
+		var subnet netip.Prefix
+		if len(nw.IPAM.Config) > 0 {
+			subnet, _ = netip.ParsePrefix(nw.IPAM.Config[0].Subnet)
+		}
+
+		if subnet.IsValid() {
+			if err = cleanupIptables(bridgeName, subnet); err != nil {
 				errs = append(errs, fmt.Errorf("cleanup iptables for Docker network '%s': %w", NetworkName, err))
 			} else {
 				slog.Info("Cleaned up iptables rules for Docker network.", "name", NetworkName, "bridge", bridgeName)
 			}
+		}
 
 		if err = m.client.NetworkRemove(ctx, NetworkName); err == nil {
 			slog.Info("Docker network removed.", "name", NetworkName)

internal/machine/network/wireguard_linux.go

@@ -81,19 +81,15 @@ func (n *WireGuardNetwork) Configure(config Config) error {
 	}
 	slog.Info("Configured WireGuard interface.", "name", n.link.Attrs().Name)
 
-	machinePrefix := netip.PrefixFrom(MachineIP(config.Subnet), config.Subnet.Bits())
 	managementPrefix, err := addrToSingleIPPrefix(config.ManagementIP)
 	if err != nil {
 		return fmt.Errorf("parse management IP: %w", err)
 	}
-	addrs := []netip.Prefix{managementPrefix, machinePrefix}
+	addrs := []netip.Prefix{managementPrefix}
 	if err = n.updateAddresses(addrs); err != nil {
 		return err
 	}
-	slog.Info(
+	slog.Info("Updated addresses of the WireGuard interface.", "name", n.link.Attrs().Name, "addrs", addrs)
-		"Updated addresses of the WireGuard interface.",
-		"name", n.link.Attrs().Name, "addrs", addrs,
-	)
 
 	// Bring the WireGuard interface up if it's not already up.
 	if n.link.Attrs().Flags&unix.IFF_UP != unix.IFF_UP {
@@ -102,7 +98,7 @@ func (n *WireGuardNetwork) Configure(config Config) error {
 		}
 		slog.Info("Brought WireGuard interface up.", "name", n.link.Attrs().Name)
 	}
-	if err = n.updatePeerRoutes(); err != nil {
+	if err = n.updatePeerRoutes(MachineIP(config.Subnet)); err != nil {
 		return err
 	}
 	slog.Info(
@@ -205,7 +201,7 @@ func (n *WireGuardNetwork) updateAddresses(addrs []netip.Prefix) error {
 
 // updatePeerRoutes adds routes to the peers via the WireGuard interface and removes old routes to peers
 // that are no longer in the configuration.
-func (n *WireGuardNetwork) updatePeerRoutes() error {
+func (n *WireGuardNetwork) updatePeerRoutes(machineIP netip.Addr) error {
 	// Build a set of compacted IP ranges for all peers.
 	var ipsetBuilder netipx.IPSetBuilder
 	for _, p := range n.peers {
@@ -225,11 +221,18 @@ func (n *WireGuardNetwork) updatePeerRoutes() error {
 	// Add routes to the computed IP ranges via the WireGuard link.
 	for _, prefix := range ipset.Prefixes() {
 		dst := prefixToIPNet(prefix)
+		var src net.IP
+		// Use the machine IP as the source address for IPv4 routes to other peers.
+		if prefix.Addr().Is4() {
+			src = machineIP.AsSlice()
+		}
+
 		if err = netlink.RouteAdd(
 			&netlink.Route{
 				LinkIndex: n.link.Attrs().Index,
 				Scope:     netlink.SCOPE_LINK,
 				Dst:       &dst,
+				Src:       src,
 			},
 		); err != nil {
 			if !errors.Is(err, unix.EEXIST) {