philipp/uncloud
1
0
Fork 0
mirror of https://github.com/psviderski/uncloud.git synced 2026-08-26 19:13:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: allow DNS queries from Uncloud containers to the embedded DNS server

Browse Source
This commit is contained in:
Pavel Sviderski
2025-05-05 20:00:34 +10:00
parent 93346301fb
commit 3648187219
6 changed files with 128 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/machine/docker/manager.go
+3 -3
View File
@@ -4,19 +4,19 @@ import (
"context"
"errors"
"fmt"
"log/slog"
"time"
dockercontainer "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/events"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/pkg/api"
"log/slog"
"time"
)
const (
NetworkName = "uncloud"
UserChain = "DOCKER-USER"
// EventsDebounceInterval defines how long to wait before processing the next Docker event. Multiple events
// occurring within this window will be processed together as a single event to prevent system overload.
EventsDebounceInterval = 100 * time.Millisecond
internal/machine/docker/manager_darwin.go
+3 -3
View File
@@ -8,7 +8,7 @@ import (
"net/netip"
)
// EnsureUncloudNetwork is a stub for darwin.
func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix) error {
return fmt.Errorf("not supported on darwin")
// EnsureUncloudNetwork is a stub for Darwin.
func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error {
return fmt.Errorf("not supported on Darwin")
}
internal/machine/docker/manager_linux.go
+36 -7
View File
@@ -5,23 +5,26 @@ import (
"fmt"
"log/slog"
"net/netip"
"strconv"
dnetwork "github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"github.com/docker/docker/libnetwork/iptables"
"github.com/psviderski/uncloud/internal/machine/dns"
"github.com/psviderski/uncloud/internal/machine/firewall"
"github.com/psviderski/uncloud/internal/machine/network"
)
// EnsureUncloudNetwork creates the Docker bridge network NetworkName with the provided machine subnet
// if it doesn't exist. If the network exists but has a different subnet, it removes and recreates the network.
// It also configures iptables to allow container access from the WireGuard network.
func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix) error {
func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix, dnsServer netip.Addr) error {
// Ensure the Docker network 'uncloud' is created with the correct subnet.
needsCreation := false
nw, err := m.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{})
if err != nil {
if !client.IsErrNotFound(err) {
return fmt.Errorf("inspect Docker network %q: %w", NetworkName, err)
return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
}
needsCreation = true
} else if nw.IPAM.Config[0].Subnet != subnet.String() {
@@ -32,7 +35,7 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix)
)
if err = m.client.NetworkRemove(ctx, NetworkName); err != nil {
// It can still fail if the network is in use by a container. Leave it to the user to resolve the issue.
return fmt.Errorf("remove Docker network %q: %w", NetworkName, err)
return fmt.Errorf("remove Docker network '%s': %w", NetworkName, err)
}
needsCreation = true
}
@@ -51,12 +54,12 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix)
},
},
); err != nil {
return fmt.Errorf("create Docker network %q: %w", NetworkName, err)
return fmt.Errorf("create Docker network '%s': %w", NetworkName, err)
}
slog.Info("Docker network created.", "name", NetworkName, "subnet", subnet.String())
if nw, err = m.client.NetworkInspect(ctx, NetworkName, dnetwork.InspectOptions{}); err != nil {
return fmt.Errorf("inspect Docker network %q: %w", NetworkName, err)
return fmt.Errorf("inspect Docker network '%s': %w", NetworkName, err)
}
}
@@ -68,10 +71,36 @@ func (m *Manager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix)
// Bridge name doesn't seem to be documented but this is the source code where it is generated:
// https://github.com/moby/moby/blob/v27.2.1/libnetwork/drivers/bridge/bridge_linux.go#L664
bridgeName := "br-" + nw.ID[:12]
if err = configureIptables(bridgeName, dnsServer); err != nil {
return fmt.Errorf("configure iptables for Docker network '%s': %w", NetworkName, err)
}
return nil
}
// configureIptables configures iptables rules for the uncloud Docker network.
func configureIptables(bridgeName string, dnsServer netip.Addr) error {
ipt := iptables.GetIptable(iptables.IPv4)
rule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"}
if err = ipt.ProgramRule(iptables.Filter, UserChain, iptables.Insert, rule); err != nil {
// Allow traffic from other machines and their containers through the WG mesh to the Uncloud containers
// on the machine.
wgRule := []string{"--in-interface", network.WireGuardInterfaceName, "--out-interface", bridgeName, "-j", "ACCEPT"}
if err := ipt.ProgramRule(iptables.Filter, firewall.DockerUserChain, iptables.Insert, wgRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
// Allow DNS queries from Uncloud containers to the embedded DNS server.
for _, proto := range []string{"udp", "tcp"} {
dnsRule := []string{
"--in-interface", bridgeName,
"--dst", dnsServer.String(),
"--protocol", proto,
"--dport", strconv.Itoa(dns.Port),
"-j", "ACCEPT",
}
if err := ipt.ProgramRule(iptables.Filter, firewall.UncloudInputChain, iptables.Insert, dnsRule); err != nil {
return fmt.Errorf("insert iptables rule: %w", err)
}
}
return nil
internal/machine/firewall/iptables_darwin.go
+8
View File
@@ -0,0 +1,8 @@
package firewall
import "fmt"
// ConfigureIptablesChains is a stub for Darwin.
func ConfigureIptablesChains() error {
return fmt.Errorf("not supported on Darwin")
}
internal/machine/firewall/iptables_linux.go
+74
View File
@@ -0,0 +1,74 @@
package firewall
import (
"fmt"
"strconv"
"strings"
"github.com/docker/docker/libnetwork/iptables"
"github.com/psviderski/uncloud/internal/machine/network"
)
const (
DockerUserChain = "DOCKER-USER"
UncloudInputChain = "UNCLOUD-INPUT"
)
// ConfigureIptablesChains sets up custom iptables chains and initial firewall rules for Uncloud networking.
func ConfigureIptablesChains() error {
// Ensure iptables UNCLOUD-INPUT chain with a RETURN rule exists. All existing rules are flushed.
ipt := iptables.GetIptable(iptables.IPv4)
if _, err := ipt.NewChain(UncloudInputChain, iptables.Filter); err != nil {
return fmt.Errorf("create iptables chain '%s': %w", UncloudInputChain, err)
}
if err := ipt.RawCombinedOutput("-t", string(iptables.Filter), "-F", UncloudInputChain); err != nil {
return fmt.Errorf("flush iptables chain '%s': %w", UncloudInputChain, err)
}
if err := ipt.AddReturnRule(UncloudInputChain); err != nil {
return fmt.Errorf("add the RETURN rule for iptables chain '%s': %w", UncloudInputChain, err)
}
// Ensure the main iptables INPUT chain has a jump rule to the UNCLOUD-INPUT chain before any DROP/REJECT rules.
jumpRule := []string{"-m", "comment", "--comment", "Uncloud-managed", "-j", UncloudInputChain}
if !ipt.Exists(iptables.Filter, "INPUT", jumpRule...) {
// Look for the first DROP/REJECT rule in the INPUT chain.
out, err := ipt.Raw("-t", string(iptables.Filter), "-L", "INPUT", "--line-numbers")
if err != nil {
return fmt.Errorf("get iptables rules for chain '%s': %w", UncloudInputChain, err)
}
firstRejectRuleNum := 0
for _, line := range strings.Split(string(out), "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
if fields[1] == "DROP" || fields[1] == "REJECT" {
if ruleNum, err := strconv.Atoi(fields[0]); err == nil {
firstRejectRuleNum = ruleNum
break
}
}
}
var addJumpRule []string
if firstRejectRuleNum > 0 {
addJumpRule = append([]string{"-t", string(iptables.Filter), "-I", "INPUT", strconv.Itoa(firstRejectRuleNum)},
jumpRule...)
} else {
addJumpRule = append([]string{"-t", string(iptables.Filter), "-A", "INPUT"}, jumpRule...)
}
if err = ipt.RawCombinedOutput(addJumpRule...); err != nil {
return fmt.Errorf("add iptables rule '%s': %w", strings.Join(addJumpRule, " "), err)
}
}
// Allow WireGuard traffic to the machine.
acceptWireGuardRule := []string{"-p", "udp", "--dport", strconv.Itoa(network.WireGuardPort), "-j", "ACCEPT"}
err := ipt.ProgramRule(iptables.Filter, UncloudInputChain, iptables.Insert, acceptWireGuardRule)
if err != nil {
return fmt.Errorf("insert iptables rule '%s': %w", strings.Join(acceptWireGuardRule, " "), err)
}
return nil
}
internal/machine/network.go
+4 -70
View File
@@ -9,17 +9,16 @@ import (
"net/netip"
"slices"
"strconv"
"strings"
"time"
"github.com/cenkalti/backoff/v4"
"github.com/docker/docker/client"
"github.com/docker/docker/libnetwork/iptables"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/caddyfile"
"github.com/psviderski/uncloud/internal/machine/corroservice"
"github.com/psviderski/uncloud/internal/machine/dns"
"github.com/psviderski/uncloud/internal/machine/docker"
"github.com/psviderski/uncloud/internal/machine/firewall"
"github.com/psviderski/uncloud/internal/machine/network"
"github.com/psviderski/uncloud/internal/machine/store"
"golang.org/x/sync/errgroup"
@@ -27,8 +26,7 @@ import (
)
const (
APIPort = 51000
iptablesUncloudInputChain = "UNCLOUD-INPUT"
APIPort = 51000
)
type networkController struct {
@@ -82,7 +80,7 @@ func newNetworkController(
}
func (nc *networkController) Run(ctx context.Context) error {
if err := nc.configureIptablesChains(); err != nil {
if err := firewall.ConfigureIptablesChains(); err != nil {
return fmt.Errorf("configure iptables chains: %w", err)
}
@@ -214,65 +212,6 @@ func (nc *networkController) Run(ctx context.Context) error {
return errGroup.Wait()
}
// configureIptablesChains sets up custom iptables chains and initial firewall rules for Uncloud networking.
func (nc *networkController) configureIptablesChains() error {
// Ensure iptables UNCLOUD-INPUT chain with a RETURN rule exists. All existing rules are flushed.
ipt := iptables.GetIptable(iptables.IPv4)
if _, err := ipt.NewChain(iptablesUncloudInputChain, iptables.Filter); err != nil {
return fmt.Errorf("create iptables chain '%s': %w", iptablesUncloudInputChain, err)
}
if err := ipt.RawCombinedOutput("-t", string(iptables.Filter), "-F", iptablesUncloudInputChain); err != nil {
return fmt.Errorf("flush iptables chain '%s': %w", iptablesUncloudInputChain, err)
}
if err := ipt.AddReturnRule(iptablesUncloudInputChain); err != nil {
return fmt.Errorf("add the RETURN rule for iptables chain '%s': %w", iptablesUncloudInputChain, err)
}
// Ensure the main iptables INPUT chain has a jump rule to the UNCLOUD-INPUT chain before any DROP/REJECT rules.
jumpRule := []string{"-m", "comment", "--comment", "Uncloud-managed", "-j", iptablesUncloudInputChain}
if !ipt.Exists(iptables.Filter, "INPUT", jumpRule...) {
// Look for the first DROP/REJECT rule in the INPUT chain.
out, err := ipt.Raw("-t", string(iptables.Filter), "-L", "INPUT", "--line-numbers")
if err != nil {
return fmt.Errorf("get iptables rules for chain '%s': %w", iptablesUncloudInputChain, err)
}
firstRejectRuleNum := 0
for _, line := range strings.Split(string(out), "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
if fields[1] == "DROP" || fields[1] == "REJECT" {
if ruleNum, err := strconv.Atoi(fields[0]); err == nil {
firstRejectRuleNum = ruleNum
break
}
}
}
var addJumpRule []string
if firstRejectRuleNum > 0 {
addJumpRule = append([]string{"-t", string(iptables.Filter), "-I", "INPUT", strconv.Itoa(firstRejectRuleNum)},
jumpRule...)
} else {
addJumpRule = append([]string{"-t", string(iptables.Filter), "-A", "INPUT"}, jumpRule...)
}
if err = ipt.RawCombinedOutput(addJumpRule...); err != nil {
return fmt.Errorf("add iptables rule '%s': %w", strings.Join(addJumpRule, " "), err)
}
}
// Allow WireGuard traffic to the machine.
acceptWireGuardRule := []string{"-p", "udp", "--dport", strconv.Itoa(network.WireGuardPort), "-j", "ACCEPT"}
err := ipt.ProgramRule(iptables.Filter, iptablesUncloudInputChain, iptables.Insert, acceptWireGuardRule)
if err != nil {
return fmt.Errorf("insert iptables rule '%s': %w", strings.Join(acceptWireGuardRule, " "), err)
}
return nil
}
// prepareAndWatchDocker configures the Docker network and watches local Docker containers to sync them
// to the cluster store.
func (nc *networkController) prepareAndWatchDocker(ctx context.Context) error {
@@ -281,16 +220,11 @@ func (nc *networkController) prepareAndWatchDocker(ctx context.Context) error {
return fmt.Errorf("wait for Docker daemon: %w", err)
}
if err := manager.EnsureUncloudNetwork(ctx, nc.state.Network.Subnet); err != nil {
if err := manager.EnsureUncloudNetwork(ctx, nc.state.Network.Subnet, nc.dnsServer.ListenAddr()); err != nil {
return fmt.Errorf("ensure Docker network: %w", err)
}
slog.Info("Docker network configured.")
// TODO: add iptables rules to UNCLOUD-INPUT to allow DNS queries from Uncloud containers
// to the embedded DNS server:
// iptables -A UNCLOUD-INPUT -d 10.210.0.1/32 -i br-f6db1df6d60b -p tcp -m tcp --dport 53 -j ACCEPT
//. iptables -A UNCLOUD-INPUT -d 10.210.0.1/32 -i br-f6db1df6d60b -p udp -m udp --dport 53 -j ACCEPT
slog.Info("Watching Docker containers and syncing them to cluster store.")
// Retry to watch and sync containers until the context is done.
boff := backoff.WithContext(backoff.NewExponentialBackOff(