add management IP to machine and peer configs, update daemon to listen on it

cmd/uncloudd/main.go

@@ -12,6 +12,8 @@ import (
 )
 
 func main() {
+	slog.SetLogLoggerLevel(slog.LevelDebug)
+
 	var dataDir string
 	cmd := &cobra.Command{
 		Use:           "uncloudd",

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54
 	go.uber.org/zap v1.27.0
+	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba
 	golang.org/x/crypto v0.26.0
 	golang.org/x/sync v0.8.0
 	golang.org/x/sys v0.24.0

go.sum

@@ -597,6 +597,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8
 go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

internal/cli/cluster.go

@@ -147,8 +147,8 @@ func (c *Cluster) AddMachine(ctx context.Context, name, user, host string, port
 		return "", err
 	}
 	userPeerCfg := network.PeerConfig{
-		Subnet:    netip.PrefixFrom(clusterUser.IPv6(), 128),
+		ManagementIP: clusterUser.ManagementIP(),
-		PublicKey: clusterUser.PublicKey(),
+		PublicKey:    clusterUser.PublicKey(),
 	}
 	mcfg, err := machine.NewBootstrapConfig(name, netip.Prefix{}, userPeerCfg)
 	if err != nil {

internal/cli/user.go

@@ -44,6 +44,6 @@ func (u *User) PublicKey() secret.Secret {
 	return pubKey[:]
 }
 
-func (u *User) IPv6() netip.Addr {
+func (u *User) ManagementIP() netip.Addr {
 	return network.PeerIPv6(u.PublicKey())
 }

internal/machine/config.go

@@ -72,10 +72,11 @@ func NewBootstrapConfig(name string, subnet netip.Prefix, peers ...network.PeerC
 		ID:   mid,
 		Name: name,
 		Network: &network.Config{
-			Subnet:     subnet,
+			Subnet:       subnet,
-			PrivateKey: privKey,
+			ManagementIP: network.PeerIPv6(pubKey),
-			PublicKey:  pubKey,
+			PrivateKey:   privKey,
-			Peers:      peers,
+			PublicKey:    pubKey,
+			Peers:        peers,
 		},
 	}, nil
 }

internal/machine/daemon/daemon.go

@@ -7,6 +7,7 @@ import (
 	"google.golang.org/grpc"
 	"log/slog"
 	"net"
+	"strconv"
 	"uncloud/internal/machine"
 	"uncloud/internal/machine/api"
 	"uncloud/internal/machine/api/pb"
@@ -40,8 +41,8 @@ func Run(ctx context.Context, dataDir string) error {
 	//}
 	//fmt.Println("Addresses:", addrs)
 
-	addr := fmt.Sprintf("127.0.0.1:%d", MachineAPIPort)
+	apiAddr := net.JoinHostPort(cfg.Network.ManagementIP.String(), strconv.Itoa(MachineAPIPort))
-	listener, err := net.Listen("tcp", addr)
+	listener, err := net.Listen("tcp", apiAddr)
 	if err != nil {
 		return fmt.Errorf("listen API port: %w", err)
 	}
@@ -51,7 +52,7 @@ func Run(ctx context.Context, dataDir string) error {
 	// Use an errgroup to coordinate error handling and graceful shutdown of multiple daemon components.
 	errGroup, ctx := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
-		slog.Info("Starting API server.", "addr", addr)
+		slog.Info("Starting API server.", "addr", apiAddr)
 		if sErr := grpcServer.Serve(listener); sErr != nil {
 			return fmt.Errorf("API server failed: %w", sErr)
 		}

internal/machine/network/config.go

@@ -16,14 +16,20 @@ var (
 type Config struct {
 	// Subnet is the IPv4 address range allocated to the machine. The machine's IP address is the first address
 	// in the subnet. Other IP addresses are allocated to containers running on the machine.
-	Subnet     netip.Prefix
+	Subnet netip.Prefix
-	PrivateKey secret.Secret
+	// ManagementIP is the ManagementIP address assigned to the machine within the WireGuard network. This address is used
-	PublicKey  secret.Secret
+	// for cluster management traffic, such as gRPC communication with the machine API server and Serf gossip.
-	Peers      []PeerConfig `json:",omitempty"`
+	ManagementIP netip.Addr
+	PrivateKey   secret.Secret
+	PublicKey    secret.Secret
+	Peers        []PeerConfig `json:",omitempty"`
 }
 
 type PeerConfig struct {
-	Subnet       netip.Prefix
+	Subnet *netip.Prefix `json:",omitempty"`
+	// ManagementIP is the ManagementIP address assigned to the peer within the WireGuard network. This address is used
+	// for cluster management traffic, such as gRPC communication with the machine API server and Serf gossip.
+	ManagementIP netip.Addr
 	Endpoint     *netip.AddrPort  `json:",omitempty"`
 	AllEndpoints []netip.AddrPort `json:",omitempty"`
 	PublicKey    secret.Secret
@@ -32,7 +38,7 @@ type PeerConfig struct {
 func (c Config) toDeviceConfig() (wgtypes.Config, error) {
 	privateKey, err := wgtypes.NewKey(c.PrivateKey)
 	if err != nil {
-		panic(fmt.Errorf("parse private key: %w", err))
+		return wgtypes.Config{}, fmt.Errorf("parse private key: %w", err)
 	}
 	listenPort := WireGuardPort
 
@@ -43,10 +49,18 @@ func (c Config) toDeviceConfig() (wgtypes.Config, error) {
 		if kErr != nil {
 			return wgtypes.Config{}, fmt.Errorf("parse peer public key: %w", kErr)
 		}
+		manageIP, mErr := addrToSingleIPPrefix(peerConfig.ManagementIP)
+		if mErr != nil {
+			return wgtypes.Config{}, fmt.Errorf("parse management IP: %w", mErr)
+		}
+		allowedIPs := []net.IPNet{prefixToIPNet(manageIP)}
+		if peerConfig.Subnet != nil {
+			allowedIPs = append(allowedIPs, prefixToIPNet(*peerConfig.Subnet))
+		}
 		wgPeerConfigs[i] = wgtypes.PeerConfig{
 			PublicKey:                   peerPublicKey,
 			ReplaceAllowedIPs:           true,
-			AllowedIPs:                  []net.IPNet{prefixToIPNet(peerConfig.Subnet)},
+			AllowedIPs:                  allowedIPs,
 			PersistentKeepaliveInterval: &persistentKeepalive,
 		}
 		if peerConfig.Endpoint != nil {

internal/machine/network/ip.go

@@ -1,6 +1,7 @@
 package network
 
 import (
+	"fmt"
 	"net"
 	"net/netip"
 	"uncloud/internal/secret"
@@ -24,3 +25,35 @@ func prefixToIPNet(prefix netip.Prefix) net.IPNet {
 		Mask: net.CIDRMask(prefix.Bits(), prefix.Addr().BitLen()),
 	}
 }
+
+// ipNetToPrefix returns a netip.Prefix from the net.IPNet type. If ipNet is invalid, ok is false.
+// Based on https://github.com/tailscale/tailscale/blob/main/net/netaddr/netaddr.go
+func ipNetToPrefix(ipNet net.IPNet) (netip.Prefix, error) {
+	ip, ok := netip.AddrFromSlice(ipNet.IP)
+	if !ok {
+		return netip.Prefix{}, fmt.Errorf("invalid IP network")
+	}
+	ip = ip.Unmap()
+
+	if l := len(ipNet.Mask); l != net.IPv4len && l != net.IPv6len {
+		return netip.Prefix{}, fmt.Errorf("invalid IP network mask length: %d", l)
+	}
+
+	ones, bits := ipNet.Mask.Size()
+	if ones == 0 && bits == 0 {
+		return netip.Prefix{}, fmt.Errorf("non-contiguous IP network mask")
+	}
+
+	return netip.PrefixFrom(ip, ones), nil
+}
+
+func addrToSingleIPPrefix(addr netip.Addr) (netip.Prefix, error) {
+	if !addr.IsValid() {
+		return netip.Prefix{}, fmt.Errorf("invalid IP address")
+	}
+	bits := 32
+	if addr.Is6() {
+		bits = 128
+	}
+	return addr.Prefix(bits)
+}

internal/machine/network/wireguard_linux.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/vishvananda/netlink"
+	"go4.org/netipx"
 	"golang.org/x/sys/unix"
 	"golang.zx2c4.com/wireguard/wgctrl"
 	"log/slog"
@@ -102,10 +103,12 @@ func (n *WireGuardNetwork) Configure(config Config) error {
 	}
 	slog.Info("Configured WireGuard interface.", "name", n.link.Attrs().Name)
 
-	machineIP := MachineIP(config.Subnet)
+	machinePrefix := netip.PrefixFrom(MachineIP(config.Subnet), config.Subnet.Bits())
-	machineIPSubnet := netip.PrefixFrom(machineIP, config.Subnet.Bits())
+	managementPrefix, err := addrToSingleIPPrefix(config.ManagementIP)
-	machineIPv6 := netip.PrefixFrom(PeerIPv6(config.PublicKey), 128)
+	if err != nil {
-	addrs := []netip.Prefix{machineIPSubnet, machineIPv6}
+		return fmt.Errorf("parse management IP: %w", err)
+	}
+	addrs := []netip.Prefix{managementPrefix, machinePrefix}
 	if err = n.updateAddresses(addrs); err != nil {
 		return err
 	}
@@ -160,39 +163,55 @@ func (n *WireGuardNetwork) updateAddresses(addrs []netip.Prefix) error {
 // updatePeerRoutes adds routes to the peers via the WireGuard interface and removes old routes to peers
 // that are no longer in the configuration.
 func (n *WireGuardNetwork) updatePeerRoutes() error {
-	// Add routes to the peers via the WireGuard link.
+	// Build a set of compacted IP ranges for all peers.
+	var ipsetBuilder netipx.IPSetBuilder
 	for _, p := range n.peers {
-		dst := prefixToIPNet(p.config.Subnet)
+		prefixes, err := p.prefixes()
-		if err := netlink.RouteAdd(&netlink.Route{
+		if err != nil {
+			return fmt.Errorf("get peer addresses: %w", err)
+		}
+		for _, pref := range prefixes {
+			ipsetBuilder.AddPrefix(pref)
+		}
+	}
+	ipset, err := ipsetBuilder.IPSet()
+	if err != nil {
+		return fmt.Errorf("build list of IP ranges for peers: %w", err)
+	}
+
+	// Add routes to the computed IP ranges via the WireGuard link.
+	for _, prefix := range ipset.Prefixes() {
+		dst := prefixToIPNet(prefix)
+		if err = netlink.RouteAdd(&netlink.Route{
 			LinkIndex: n.link.Attrs().Index,
 			Scope:     netlink.SCOPE_LINK,
 			Dst:       &dst,
 		}); err != nil && !errors.Is(err, unix.EEXIST) {
 			return fmt.Errorf("add route to WireGuard link %q: %w", n.link.Attrs().Name, err)
 		}
-		slog.Debug("Added route to peer via WireGuard interface.",
+		slog.Debug("Added route to peer(s) via WireGuard interface.",
-			"name", n.link.Attrs().Name, "peer", dst)
+			"name", n.link.Attrs().Name, "dst", prefix)
 	}
-	// Remove old routes to peers that are no longer in the configuration.
+
+	// Remove old routes to IP ranges that are no longer in the configuration.
+	addedRoutes := ipset.Prefixes()
 	routes, err := netlink.RouteList(n.link, netlink.FAMILY_ALL)
 	if err != nil {
 		return fmt.Errorf("list routes on WireGuard link %q: %w", n.link.Attrs().Name, err)
 	}
 	for _, route := range routes {
-		old := true
+		routePrefix, pErr := ipNetToPrefix(*route.Dst)
-		for _, p := range n.peers {
+		if pErr != nil {
-			if route.Dst.String() == p.config.Subnet.String() {
+			return fmt.Errorf("parse route destination: %w", pErr)
-				old = false
-				break
-			}
 		}
-		if old {
+		if slices.Contains(addedRoutes, routePrefix) {
-			if err = netlink.RouteDel(&route); err != nil {
+			continue
-				return fmt.Errorf("remove route %q from WireGuard link %q: %w", route.Dst, n.link.Attrs().Name, err)
-			}
-			slog.Debug("Removed route to peer via WireGuard interface.",
-				"name", n.link.Attrs().Name, "peer", route.Dst)
 		}
+		if err = netlink.RouteDel(&route); err != nil {
+			return fmt.Errorf("remove route %q from WireGuard link %q: %w", route.Dst, n.link.Attrs().Name, err)
+		}
+		slog.Debug("Removed route to peer(s) via WireGuard interface.",
+			"name", n.link.Attrs().Name, "dst", routePrefix)
 	}
 	return nil
 }
@@ -201,3 +220,15 @@ func (n *WireGuardNetwork) Run(ctx context.Context) error {
 	<-ctx.Done()
 	return nil
 }
+
+func (p peer) prefixes() ([]netip.Prefix, error) {
+	managePrefix, err := addrToSingleIPPrefix(p.config.ManagementIP)
+	if err != nil {
+		return nil, fmt.Errorf("parse management IP: %w", err)
+	}
+	prefixes := []netip.Prefix{managePrefix}
+	if p.config.Subnet != nil {
+		prefixes = append(prefixes, *p.config.Subnet)
+	}
+	return prefixes, nil
+}