From fd396dd91f0f1576a56f51c839ce61fe88c67113 Mon Sep 17 00:00:00 2001 From: Pavel Sviderski Date: Fri, 30 Aug 2024 18:33:37 +1000 Subject: [PATCH] add management IP to machine and peer configs, update daemon to listen on it --- cmd/uncloudd/main.go | 2 + go.mod | 1 + go.sum | 2 + internal/cli/cluster.go | 4 +- internal/cli/user.go | 2 +- internal/machine/config.go | 9 +-- internal/machine/daemon/daemon.go | 7 +- internal/machine/network/config.go | 28 ++++++-- internal/machine/network/ip.go | 33 +++++++++ internal/machine/network/wireguard_linux.go | 75 +++++++++++++++------ 10 files changed, 124 insertions(+), 39 deletions(-) diff --git a/cmd/uncloudd/main.go b/cmd/uncloudd/main.go index 05ecdb37..806692f3 100644 --- a/cmd/uncloudd/main.go +++ b/cmd/uncloudd/main.go @@ -12,6 +12,8 @@ import ( ) func main() { + slog.SetLogLoggerLevel(slog.LevelDebug) + var dataDir string cmd := &cobra.Command{ Use: "uncloudd", diff --git a/go.mod b/go.mod index 8307e0b9..7a0ee17d 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54 go.uber.org/zap v1.27.0 + go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.26.0 golang.org/x/sync v0.8.0 golang.org/x/sys v0.24.0 diff --git a/go.sum b/go.sum index 924d69f2..6e9eb114 100644 --- a/go.sum +++ b/go.sum @@ -597,6 +597,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/internal/cli/cluster.go b/internal/cli/cluster.go index 8e13aa0a..aba00de8 100644 --- a/internal/cli/cluster.go +++ b/internal/cli/cluster.go @@ -147,8 +147,8 @@ func (c *Cluster) AddMachine(ctx context.Context, name, user, host string, port return "", err } userPeerCfg := network.PeerConfig{ - Subnet: netip.PrefixFrom(clusterUser.IPv6(), 128), - PublicKey: clusterUser.PublicKey(), + ManagementIP: clusterUser.ManagementIP(), + PublicKey: clusterUser.PublicKey(), } mcfg, err := machine.NewBootstrapConfig(name, netip.Prefix{}, userPeerCfg) if err != nil { diff --git a/internal/cli/user.go b/internal/cli/user.go index ed91d899..b33f85c5 100644 --- a/internal/cli/user.go +++ b/internal/cli/user.go @@ -44,6 +44,6 @@ func (u *User) PublicKey() secret.Secret { return pubKey[:] } -func (u *User) IPv6() netip.Addr { +func (u *User) ManagementIP() netip.Addr { return network.PeerIPv6(u.PublicKey()) } diff --git a/internal/machine/config.go b/internal/machine/config.go index ac719e70..3ce4e67c 100644 --- a/internal/machine/config.go +++ b/internal/machine/config.go @@ -72,10 +72,11 @@ func NewBootstrapConfig(name string, subnet netip.Prefix, peers ...network.PeerC ID: mid, Name: name, Network: &network.Config{ - Subnet: subnet, - PrivateKey: privKey, - PublicKey: pubKey, - Peers: peers, + Subnet: subnet, + ManagementIP: network.PeerIPv6(pubKey), + PrivateKey: privKey, + PublicKey: pubKey, + Peers: peers, }, }, nil } diff --git a/internal/machine/daemon/daemon.go b/internal/machine/daemon/daemon.go index 09beb17a..5c77999a 100644 --- a/internal/machine/daemon/daemon.go +++ b/internal/machine/daemon/daemon.go @@ -7,6 +7,7 @@ import ( "google.golang.org/grpc" "log/slog" "net" + "strconv" "uncloud/internal/machine" "uncloud/internal/machine/api" "uncloud/internal/machine/api/pb" @@ -40,8 +41,8 @@ func Run(ctx context.Context, dataDir string) error { //} //fmt.Println("Addresses:", addrs) - addr := fmt.Sprintf("127.0.0.1:%d", MachineAPIPort) - listener, err := net.Listen("tcp", addr) + apiAddr := net.JoinHostPort(cfg.Network.ManagementIP.String(), strconv.Itoa(MachineAPIPort)) + listener, err := net.Listen("tcp", apiAddr) if err != nil { return fmt.Errorf("listen API port: %w", err) } @@ -51,7 +52,7 @@ func Run(ctx context.Context, dataDir string) error { // Use an errgroup to coordinate error handling and graceful shutdown of multiple daemon components. errGroup, ctx := errgroup.WithContext(ctx) errGroup.Go(func() error { - slog.Info("Starting API server.", "addr", addr) + slog.Info("Starting API server.", "addr", apiAddr) if sErr := grpcServer.Serve(listener); sErr != nil { return fmt.Errorf("API server failed: %w", sErr) } diff --git a/internal/machine/network/config.go b/internal/machine/network/config.go index e96a0f21..5ee65963 100644 --- a/internal/machine/network/config.go +++ b/internal/machine/network/config.go @@ -16,14 +16,20 @@ var ( type Config struct { // Subnet is the IPv4 address range allocated to the machine. The machine's IP address is the first address // in the subnet. Other IP addresses are allocated to containers running on the machine. - Subnet netip.Prefix - PrivateKey secret.Secret - PublicKey secret.Secret - Peers []PeerConfig `json:",omitempty"` + Subnet netip.Prefix + // ManagementIP is the ManagementIP address assigned to the machine within the WireGuard network. This address is used + // for cluster management traffic, such as gRPC communication with the machine API server and Serf gossip. + ManagementIP netip.Addr + PrivateKey secret.Secret + PublicKey secret.Secret + Peers []PeerConfig `json:",omitempty"` } type PeerConfig struct { - Subnet netip.Prefix + Subnet *netip.Prefix `json:",omitempty"` + // ManagementIP is the ManagementIP address assigned to the peer within the WireGuard network. This address is used + // for cluster management traffic, such as gRPC communication with the machine API server and Serf gossip. + ManagementIP netip.Addr Endpoint *netip.AddrPort `json:",omitempty"` AllEndpoints []netip.AddrPort `json:",omitempty"` PublicKey secret.Secret @@ -32,7 +38,7 @@ type PeerConfig struct { func (c Config) toDeviceConfig() (wgtypes.Config, error) { privateKey, err := wgtypes.NewKey(c.PrivateKey) if err != nil { - panic(fmt.Errorf("parse private key: %w", err)) + return wgtypes.Config{}, fmt.Errorf("parse private key: %w", err) } listenPort := WireGuardPort @@ -43,10 +49,18 @@ func (c Config) toDeviceConfig() (wgtypes.Config, error) { if kErr != nil { return wgtypes.Config{}, fmt.Errorf("parse peer public key: %w", kErr) } + manageIP, mErr := addrToSingleIPPrefix(peerConfig.ManagementIP) + if mErr != nil { + return wgtypes.Config{}, fmt.Errorf("parse management IP: %w", mErr) + } + allowedIPs := []net.IPNet{prefixToIPNet(manageIP)} + if peerConfig.Subnet != nil { + allowedIPs = append(allowedIPs, prefixToIPNet(*peerConfig.Subnet)) + } wgPeerConfigs[i] = wgtypes.PeerConfig{ PublicKey: peerPublicKey, ReplaceAllowedIPs: true, - AllowedIPs: []net.IPNet{prefixToIPNet(peerConfig.Subnet)}, + AllowedIPs: allowedIPs, PersistentKeepaliveInterval: &persistentKeepalive, } if peerConfig.Endpoint != nil { diff --git a/internal/machine/network/ip.go b/internal/machine/network/ip.go index d3830a49..24c93d5f 100644 --- a/internal/machine/network/ip.go +++ b/internal/machine/network/ip.go @@ -1,6 +1,7 @@ package network import ( + "fmt" "net" "net/netip" "uncloud/internal/secret" @@ -24,3 +25,35 @@ func prefixToIPNet(prefix netip.Prefix) net.IPNet { Mask: net.CIDRMask(prefix.Bits(), prefix.Addr().BitLen()), } } + +// ipNetToPrefix returns a netip.Prefix from the net.IPNet type. If ipNet is invalid, ok is false. +// Based on https://github.com/tailscale/tailscale/blob/main/net/netaddr/netaddr.go +func ipNetToPrefix(ipNet net.IPNet) (netip.Prefix, error) { + ip, ok := netip.AddrFromSlice(ipNet.IP) + if !ok { + return netip.Prefix{}, fmt.Errorf("invalid IP network") + } + ip = ip.Unmap() + + if l := len(ipNet.Mask); l != net.IPv4len && l != net.IPv6len { + return netip.Prefix{}, fmt.Errorf("invalid IP network mask length: %d", l) + } + + ones, bits := ipNet.Mask.Size() + if ones == 0 && bits == 0 { + return netip.Prefix{}, fmt.Errorf("non-contiguous IP network mask") + } + + return netip.PrefixFrom(ip, ones), nil +} + +func addrToSingleIPPrefix(addr netip.Addr) (netip.Prefix, error) { + if !addr.IsValid() { + return netip.Prefix{}, fmt.Errorf("invalid IP address") + } + bits := 32 + if addr.Is6() { + bits = 128 + } + return addr.Prefix(bits) +} diff --git a/internal/machine/network/wireguard_linux.go b/internal/machine/network/wireguard_linux.go index 33be241e..1b21aabe 100644 --- a/internal/machine/network/wireguard_linux.go +++ b/internal/machine/network/wireguard_linux.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "github.com/vishvananda/netlink" + "go4.org/netipx" "golang.org/x/sys/unix" "golang.zx2c4.com/wireguard/wgctrl" "log/slog" @@ -102,10 +103,12 @@ func (n *WireGuardNetwork) Configure(config Config) error { } slog.Info("Configured WireGuard interface.", "name", n.link.Attrs().Name) - machineIP := MachineIP(config.Subnet) - machineIPSubnet := netip.PrefixFrom(machineIP, config.Subnet.Bits()) - machineIPv6 := netip.PrefixFrom(PeerIPv6(config.PublicKey), 128) - addrs := []netip.Prefix{machineIPSubnet, machineIPv6} + machinePrefix := netip.PrefixFrom(MachineIP(config.Subnet), config.Subnet.Bits()) + managementPrefix, err := addrToSingleIPPrefix(config.ManagementIP) + if err != nil { + return fmt.Errorf("parse management IP: %w", err) + } + addrs := []netip.Prefix{managementPrefix, machinePrefix} if err = n.updateAddresses(addrs); err != nil { return err } @@ -160,39 +163,55 @@ func (n *WireGuardNetwork) updateAddresses(addrs []netip.Prefix) error { // updatePeerRoutes adds routes to the peers via the WireGuard interface and removes old routes to peers // that are no longer in the configuration. func (n *WireGuardNetwork) updatePeerRoutes() error { - // Add routes to the peers via the WireGuard link. + // Build a set of compacted IP ranges for all peers. + var ipsetBuilder netipx.IPSetBuilder for _, p := range n.peers { - dst := prefixToIPNet(p.config.Subnet) - if err := netlink.RouteAdd(&netlink.Route{ + prefixes, err := p.prefixes() + if err != nil { + return fmt.Errorf("get peer addresses: %w", err) + } + for _, pref := range prefixes { + ipsetBuilder.AddPrefix(pref) + } + } + ipset, err := ipsetBuilder.IPSet() + if err != nil { + return fmt.Errorf("build list of IP ranges for peers: %w", err) + } + + // Add routes to the computed IP ranges via the WireGuard link. + for _, prefix := range ipset.Prefixes() { + dst := prefixToIPNet(prefix) + if err = netlink.RouteAdd(&netlink.Route{ LinkIndex: n.link.Attrs().Index, Scope: netlink.SCOPE_LINK, Dst: &dst, }); err != nil && !errors.Is(err, unix.EEXIST) { return fmt.Errorf("add route to WireGuard link %q: %w", n.link.Attrs().Name, err) } - slog.Debug("Added route to peer via WireGuard interface.", - "name", n.link.Attrs().Name, "peer", dst) + slog.Debug("Added route to peer(s) via WireGuard interface.", + "name", n.link.Attrs().Name, "dst", prefix) } - // Remove old routes to peers that are no longer in the configuration. + + // Remove old routes to IP ranges that are no longer in the configuration. + addedRoutes := ipset.Prefixes() routes, err := netlink.RouteList(n.link, netlink.FAMILY_ALL) if err != nil { return fmt.Errorf("list routes on WireGuard link %q: %w", n.link.Attrs().Name, err) } for _, route := range routes { - old := true - for _, p := range n.peers { - if route.Dst.String() == p.config.Subnet.String() { - old = false - break - } + routePrefix, pErr := ipNetToPrefix(*route.Dst) + if pErr != nil { + return fmt.Errorf("parse route destination: %w", pErr) } - if old { - if err = netlink.RouteDel(&route); err != nil { - return fmt.Errorf("remove route %q from WireGuard link %q: %w", route.Dst, n.link.Attrs().Name, err) - } - slog.Debug("Removed route to peer via WireGuard interface.", - "name", n.link.Attrs().Name, "peer", route.Dst) + if slices.Contains(addedRoutes, routePrefix) { + continue } + if err = netlink.RouteDel(&route); err != nil { + return fmt.Errorf("remove route %q from WireGuard link %q: %w", route.Dst, n.link.Attrs().Name, err) + } + slog.Debug("Removed route to peer(s) via WireGuard interface.", + "name", n.link.Attrs().Name, "dst", routePrefix) } return nil } @@ -201,3 +220,15 @@ func (n *WireGuardNetwork) Run(ctx context.Context) error { <-ctx.Done() return nil } + +func (p peer) prefixes() ([]netip.Prefix, error) { + managePrefix, err := addrToSingleIPPrefix(p.config.ManagementIP) + if err != nil { + return nil, fmt.Errorf("parse management IP: %w", err) + } + prefixes := []netip.Prefix{managePrefix} + if p.config.Subnet != nil { + prefixes = append(prefixes, *p.config.Subnet) + } + return prefixes, nil +}