philipp/uncloud
1
0
Fork 0
mirror of https://github.com/psviderski/uncloud.git synced 2026-08-26 11:03:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: do not masquerade container IPs when communicating via WG mesh

Browse Source
This commit is contained in:
Pasha Sviderski
2025-08-04 19:09:38 +10:00
parent 35d0a90125
commit b39585df4c
3 changed files with 79 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/machine/cluster.go
+16 -7
View File
@@ -94,6 +94,11 @@ func (cc *clusterController) Run(ctx context.Context) error {
return fmt.Errorf("configure iptables chains: %w", err)
}
if err := cc.ensureDockerNetwork(ctx); err != nil {
return err
}
slog.Info("Docker network configured.")
if err := cc.wgnet.Configure(*cc.state.Network); err != nil {
return fmt.Errorf("configure WireGuard network: %w", err)
}
@@ -138,6 +143,7 @@ func (cc *clusterController) Run(ctx context.Context) error {
return nil
})
// The Docker network must be created before starting the DNS server because it listens on the machine IP.
errGroup.Go(func() error {
slog.Info("Starting embedded DNS server.")
if err := cc.dnsServer.Run(ctx); err != nil {
@@ -146,9 +152,10 @@ func (cc *clusterController) Run(ctx context.Context) error {
return nil
})
// Setup Docker network and synchronise containers to the cluster store.
// Synchronise Docker containers to the cluster store.
errGroup.Go(func() error {
return cc.prepareAndWatchDocker(ctx)
slog.Info("Watching Docker containers and syncing them to cluster store.")
return cc.syncDockerContainers(ctx)
})
// Handle machine changes in the cluster. Handling machine and endpoint changes should be done
@@ -229,9 +236,8 @@ func (cc *clusterController) Run(ctx context.Context) error {
return err
}
// prepareAndWatchDocker configures the Docker network and watches local Docker containers to sync them
// to the cluster store.
func (cc *clusterController) prepareAndWatchDocker(ctx context.Context) error {
// ensureDockerNetwork ensures that the Docker network is configured and ready for containers.
func (cc *clusterController) ensureDockerNetwork(ctx context.Context) error {
if err := cc.dockerManager.WaitDaemonReady(ctx); err != nil {
return fmt.Errorf("wait for Docker daemon: %w", err)
}
@@ -243,12 +249,15 @@ func (cc *clusterController) prepareAndWatchDocker(ctx context.Context) error {
); err != nil {
return fmt.Errorf("ensure Docker network: %w", err)
}
slog.Info("Docker network configured.")
// Signal that Docker is ready for containers.
close(cc.dockerReady)
slog.Info("Watching Docker containers and syncing them to cluster store.")
return nil
}
// syncDockerContainers watches local Docker containers and syncs them to the cluster store.
func (cc *clusterController) syncDockerContainers(ctx context.Context) error {
// Retry to watch and sync containers until the context is done.
boff := backoff.WithContext(backoff.NewExponentialBackOff(
backoff.WithInitialInterval(100*time.Millisecond),
internal/machine/docker/manager_linux.go
+52 -9
View File
@@ -84,7 +84,7 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix,
// https://github.com/moby/moby/blob/v27.2.1/libnetwork/drivers/bridge/bridge_linux.go#L664
bridgeName := "br-" + nw.ID[:12]
if err = configureIptables(bridgeName, dnsServer); err != nil {
if err = configureIptables(bridgeName, subnet, dnsServer); err != nil {
return fmt.Errorf("configure iptables for Docker network '%s': %w", NetworkName, err)
}
@@ -92,11 +92,15 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix,
}
// configureIptables configures iptables rules for the uncloud Docker network.
func configureIptables(bridgeName string, dnsServer netip.Addr) error {
func configureIptables(bridgeName string, subnet netip.Prefix, dnsServer netip.Addr) error {
ipt := iptables.GetIptable(iptables.IPv4)
// Allow traffic from other machines and their containers through the WG mesh to the Uncloud containers
// on the machine.
wgRule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"}
wgRule := []string{
"--in-interface", network.WireGuardInterfaceName,
"--out-interface", bridgeName,
"-j", "ACCEPT",
}
if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Insert, wgRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
@@ -115,17 +119,49 @@ func configureIptables(bridgeName string, dnsServer netip.Addr) error {
}
}
// Skip masquerading for the container traffic going from the uncloud Docker network through the WG mesh.
// https://uncloud.run/blog/connect-docker-containers-across-hosts-wireguard#step-3-configure-ip-routing
skipMasqueradeRule := []string{
"--src", subnet.String(),
"--out-interface", network.WireGuardInterfaceName,
"-j", "RETURN",
}
// Delete and reinsert the rule to ensure it's at the top of the POSTROUTING chain before the MASQUERADE rule
// added by Docker: POSTROUTING -s 10.210.X.0/24 ! -o br-XXX -j MASQUERADE
if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil {
return fmt.Errorf("delete iptables rule: %w", err)
}
if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Insert, skipMasqueradeRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
return nil
}
// cleanupIptables deletes the iptables rules for the uncloud Docker network.
func cleanupIptables(bridgeName string) error {
func cleanupIptables(bridgeName string, subnet netip.Prefix) error {
ipt := iptables.GetIptable(iptables.IPv4)
// Delete the rule allowing traffic from the WireGuard network to the Docker bridge.
wgRule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"}
wgRule := []string{
"--in-interface", network.WireGuardInterfaceName,
"--out-interface", bridgeName,
"-j", "ACCEPT",
}
if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Delete, wgRule); err != nil {
return fmt.Errorf("delete iptables rule: %w", err)
}
// Delete the rule that skips masquerading for the container traffic going from the uncloud Docker network
// through the WG mesh.
skipMasqueradeRule := []string{
"--src", subnet.String(),
"--out-interface", network.WireGuardInterfaceName,
"-j", "RETURN",
}
if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil {
return fmt.Errorf("delete iptables rule: %w", err)
}
// Rules in uncloud-owned chains will be automatically cleaned up by the machine cleanup.
return nil
@@ -172,10 +208,17 @@ func (m *Manager) Cleanup() error {
nw, err := m.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{})
if err == nil {
bridgeName := "br-" + nw.ID[:12]
if err = cleanupIptables(bridgeName); err != nil {
errs = append(errs, fmt.Errorf("cleanup iptables for Docker network '%s': %w", NetworkName, err))
} else {
slog.Info("Cleaned up iptables rules for Docker network.", "name", NetworkName, "bridge", bridgeName)
var subnet netip.Prefix
if len(nw.IPAM.Config) > 0 {
subnet, _ = netip.ParsePrefix(nw.IPAM.Config[0].Subnet)
}
if subnet.IsValid() {
if err = cleanupIptables(bridgeName, subnet); err != nil {
errs = append(errs, fmt.Errorf("cleanup iptables for Docker network '%s': %w", NetworkName, err))
} else {
slog.Info("Cleaned up iptables rules for Docker network.", "name", NetworkName, "bridge", bridgeName)
}
}
if err = m.client.NetworkRemove(ctx, NetworkName); err == nil {
internal/machine/network/wireguard_linux.go
+11 -8
View File
@@ -81,19 +81,15 @@ func (n *WireGuardNetwork) Configure(config Config) error {
}
slog.Info("Configured WireGuard interface.", "name", n.link.Attrs().Name)
machinePrefix := netip.PrefixFrom(MachineIP(config.Subnet), config.Subnet.Bits())
managementPrefix, err := addrToSingleIPPrefix(config.ManagementIP)
if err != nil {
return fmt.Errorf("parse management IP: %w", err)
}
addrs := []netip.Prefix{managementPrefix, machinePrefix}
addrs := []netip.Prefix{managementPrefix}
if err = n.updateAddresses(addrs); err != nil {
return err
}
slog.Info(
"Updated addresses of the WireGuard interface.",
"name", n.link.Attrs().Name, "addrs", addrs,
)
slog.Info("Updated addresses of the WireGuard interface.", "name", n.link.Attrs().Name, "addrs", addrs)
// Bring the WireGuard interface up if it's not already up.
if n.link.Attrs().Flags&unix.IFF_UP != unix.IFF_UP {
@@ -102,7 +98,7 @@ func (n *WireGuardNetwork) Configure(config Config) error {
}
slog.Info("Brought WireGuard interface up.", "name", n.link.Attrs().Name)
}
if err = n.updatePeerRoutes(); err != nil {
if err = n.updatePeerRoutes(MachineIP(config.Subnet)); err != nil {
return err
}
slog.Info(
@@ -205,7 +201,7 @@ func (n *WireGuardNetwork) updateAddresses(addrs []netip.Prefix) error {
// updatePeerRoutes adds routes to the peers via the WireGuard interface and removes old routes to peers
// that are no longer in the configuration.
func (n *WireGuardNetwork) updatePeerRoutes() error {
func (n *WireGuardNetwork) updatePeerRoutes(machineIP netip.Addr) error {
// Build a set of compacted IP ranges for all peers.
var ipsetBuilder netipx.IPSetBuilder
for _, p := range n.peers {
@@ -225,11 +221,18 @@ func (n *WireGuardNetwork) updatePeerRoutes() error {
// Add routes to the computed IP ranges via the WireGuard link.
for _, prefix := range ipset.Prefixes() {
dst := prefixToIPNet(prefix)
var src net.IP
// Use the machine IP as the source address for IPv4 routes to other peers.
if prefix.Addr().Is4() {
src = machineIP.AsSlice()
}
if err = netlink.RouteAdd(
&netlink.Route{
LinkIndex: n.link.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: &dst,
Src: src,
},
); err != nil {
if !errors.Is(err, unix.EEXIST) {