chore: store ServiceContainer (includes service spec) instead of Container in Corrosion store

2026-08-26 19:13:34 +00:00 · 2025-08-14 14:53:33 +10:00
parent 9186d31d12
commit 4cc1e556dd
10 changed files with 174 additions and 95 deletions
@@ -0,0 +1,234 @@
+package docker
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/netip"
+	"strconv"
+
+	dockercontainer "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	dnetwork "github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/client"
+	"github.com/docker/docker/libnetwork/iptables"
+	"github.com/psviderski/uncloud/internal/machine/dns"
+	"github.com/psviderski/uncloud/internal/machine/firewall"
+	"github.com/psviderski/uncloud/internal/machine/network"
+	"github.com/psviderski/uncloud/pkg/api"
+)
+
+// EnsureUncloudNetwork creates the Docker bridge network NetworkName with the provided machine subnet
+// if it doesn't exist. If the network exists but has a different subnet, it removes and recreates the network.
+// It also configures iptables to allow container access from the WireGuard network.
+func (c *Controller) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error {
+	// Ensure the Docker network 'uncloud' is created with the correct subnet.
+	needsCreation := false
+	nw, err := c.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{})
+	if err != nil {
+		if !client.IsErrNotFound(err) {
+			return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
+		}
+		needsCreation = true
+	} else if nw.IPAM.Config[0].Subnet != subnet.String() {
+		// Remove the Docker network if the subnet is different.
+		// It could be a leftover from a previous incomplete cleanup.
+		slog.Info(
+			"Removing Docker network with old subnet.", "name", NetworkName, "subnet", nw.IPAM.Config[0].Subnet,
+		)
+		if err = c.client.NetworkRemove(ctx, NetworkName); err != nil {
+			// It can still fail if the network is in use by a container. Leave it to the user to resolve the issue.
+			return fmt.Errorf("remove Docker network '%s': %w", NetworkName, err)
+		}
+		needsCreation = true
+	}
+
+	if needsCreation {
+		if _, err = c.client.NetworkCreate(
+			ctx, NetworkName, dnetwork.CreateOptions{
+				Driver: "bridge",
+				Scope:  "local",
+				IPAM: &dnetwork.IPAM{
+					Config: []dnetwork.IPAMConfig{
+						{
+							Subnet: subnet.String(),
+						},
+					},
+				},
+				Labels: map[string]string{
+					api.LabelManaged: "",
+				},
+				Options: map[string]string{
+					// Starting with Docker 28.2.0 (https://github.com/moby/moby/pull/49832), we have to explicitly
+					// allow direct routing from the WireGuard interface to the bridge network.
+					"com.docker.network.bridge.trusted_host_interfaces": network.WireGuardInterfaceName,
+				},
+			},
+		); err != nil {
+			return fmt.Errorf("create Docker network '%s': %w", NetworkName, err)
+		}
+		slog.Info("Docker network created.", "name", NetworkName, "subnet", subnet.String())
+
+		if nw, err = c.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{}); err != nil {
+			return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
+		}
+	}
+
+	// Configure iptables to allow WireGuard network to access containers. The Docker daemon should have already
+	// created the DOCKER-USER chain at this point.
+	// TODO: check if this works when firewalld used instead of raw iptables. The Docker daemon has a different
+	//  code path for firewalld.
+
+	// Bridge name doesn't seem to be documented but this is the source code where it is generated:
+	// https://github.com/moby/moby/blob/v27.2.1/libnetwork/drivers/bridge/bridge_linux.go#L664
+	bridgeName := "br-" + nw.ID[:12]
+
+	if err = configureIptables(bridgeName, subnet, dnsServer); err != nil {
+		return fmt.Errorf("configure iptables for Docker network '%s': %w", NetworkName, err)
+	}
+
+	return nil
+}
+
+// configureIptables configures iptables rules for the uncloud Docker network.
+func configureIptables(bridgeName string, subnet netip.Prefix, dnsServer netip.Addr) error {
+	ipt := iptables.GetIptable(iptables.IPv4)
+	// Allow traffic from other machines and their containers through the WG mesh to the Uncloud containers
+	// on the machine.
+	wgRule := []string{
+		"--in-interface", network.WireGuardInterfaceName,
+		"--out-interface", bridgeName,
+		"-j", "ACCEPT",
+	}
+	if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Insert, wgRule); err != nil {
+		return fmt.Errorf("insert iptables rule: %w", err)
+	}
+
+	// Allow DNS queries from Uncloud containers to the embedded DNS server.
+	for _, proto := range []string{"udp", "tcp"} {
+		dnsRule := []string{
+			"--in-interface", bridgeName,
+			"--dst", dnsServer.String(),
+			"--protocol", proto,
+			"--dport", strconv.Itoa(dns.Port),
+			"-j", "ACCEPT",
+		}
+		if err := ipt.ProgramRule(iptables.Filter, firewall.UncloudInputChain, iptables.Insert, dnsRule); err != nil {
+			return fmt.Errorf("insert iptables rule: %w", err)
+		}
+	}
+
+	// Skip masquerading for the container traffic going from the uncloud Docker network through the WG mesh.
+	// https://uncloud.run/blog/connect-docker-containers-across-hosts-wireguard#step-3-configure-ip-routing
+	skipMasqueradeRule := []string{
+		"--src", subnet.String(),
+		"--out-interface", network.WireGuardInterfaceName,
+		"-j", "RETURN",
+	}
+	// Delete and reinsert the rule to ensure it's at the top of the POSTROUTING chain before the MASQUERADE rule
+	// added by Docker: POSTROUTING -s 10.210.X.0/24 ! -o br-XXX -j MASQUERADE
+	if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil {
+		return fmt.Errorf("delete iptables rule: %w", err)
+	}
+	if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Insert, skipMasqueradeRule); err != nil {
+		return fmt.Errorf("insert iptables rule: %w", err)
+	}
+
+	return nil
+}
+
+// cleanupIptables deletes the iptables rules for the uncloud Docker network.
+func cleanupIptables(bridgeName string, subnet netip.Prefix) error {
+	ipt := iptables.GetIptable(iptables.IPv4)
+	// Delete the rule allowing traffic from the WireGuard network to the Docker bridge.
+	wgRule := []string{
+		"--in-interface", network.WireGuardInterfaceName,
+		"--out-interface", bridgeName,
+		"-j", "ACCEPT",
+	}
+	if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Delete, wgRule); err != nil {
+		return fmt.Errorf("delete iptables rule: %w", err)
+	}
+
+	// Delete the rule that skips masquerading for the container traffic going from the uncloud Docker network
+	// through the WG mesh.
+	skipMasqueradeRule := []string{
+		"--src", subnet.String(),
+		"--out-interface", network.WireGuardInterfaceName,
+		"-j", "RETURN",
+	}
+	if err := ipt.ProgramRule(iptables.Nat, "POSTROUTING", iptables.Delete, skipMasqueradeRule); err != nil {
+		return fmt.Errorf("delete iptables rule: %w", err)
+	}
+
+	// Rules in uncloud-owned chains will be automatically cleaned up by the machine cleanup.
+
+	return nil
+}
+
+// Cleanup removes all uncloud-managed containers and the uncloud Docker network.
+func (c *Controller) Cleanup() error {
+	ctx := context.Background()
+	var errs []error
+
+	// Remove uncloud-managed Docker containers.
+	containers, err := c.client.ContainerList(ctx, dockercontainer.ListOptions{
+		All: true, // Include stopped containers.
+		Filters: filters.NewArgs(
+			filters.Arg("label", api.LabelManaged),
+		),
+	})
+	if err != nil {
+		errs = append(errs, fmt.Errorf("list uncloud-managed Docker containers: %w", err))
+	} else if len(containers) > 0 {
+		slog.Info("Removing uncloud-managed Docker containers.", "count", len(containers))
+		removed := 0
+
+		for _, ctr := range containers {
+			err = c.client.ContainerStop(ctx, ctr.ID, dockercontainer.StopOptions{})
+			if err != nil && !client.IsErrNotFound(err) {
+				errs = append(errs, fmt.Errorf("stop container '%s': %w", ctr.ID, err))
+			}
+
+			err = c.client.ContainerRemove(ctx, ctr.ID, dockercontainer.RemoveOptions{
+				// Remove anonymous volumes created by the container.
+				RemoveVolumes: true,
+			})
+			if err == nil {
+				removed++
+			} else if !client.IsErrNotFound(err) {
+				errs = append(errs, fmt.Errorf("remove container '%s': %w", ctr.ID, err))
+			}
+		}
+		slog.Info("Removed uncloud-managed Docker containers.", "count", removed)
+	}
+
+	// Remove the uncloud Docker network and related iptables rules.
+	nw, err := c.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{})
+	if err == nil {
+		bridgeName := "br-" + nw.ID[:12]
+		var subnet netip.Prefix
+		if len(nw.IPAM.Config) > 0 {
+			subnet, _ = netip.ParsePrefix(nw.IPAM.Config[0].Subnet)
+		}
+
+		if subnet.IsValid() {
+			if err = cleanupIptables(bridgeName, subnet); err != nil {
+				errs = append(errs, fmt.Errorf("cleanup iptables for Docker network '%s': %w", NetworkName, err))
+			} else {
+				slog.Info("Cleaned up iptables rules for Docker network.", "name", NetworkName, "bridge", bridgeName)
+			}
+		}
+
+		if err = c.client.NetworkRemove(ctx, NetworkName); err == nil {
+			slog.Info("Docker network removed.", "name", NetworkName)
+		} else if !client.IsErrNotFound(err) {
+			errs = append(errs, fmt.Errorf("remove Docker network '%s': %w", NetworkName, err))
+		}
+	} else if !client.IsErrNotFound(err) {
+		errs = append(errs, fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err))
+	}
+
+	return errors.Join(errs...)
+}