From 19d53c4f6724621b906a8d3144359fb6fce597d9 Mon Sep 17 00:00:00 2001 From: Pavel Sviderski Date: Sat, 16 Nov 2024 20:25:34 +1000 Subject: [PATCH] move Docker management from network controller to standalone DockerManager --- internal/machine/api/proxy/local.go | 3 - internal/machine/docker.go | 55 +++++++++++++++++++ internal/machine/docker_darwin.go | 14 +++++ .../{network_linux.go => docker_linux.go} | 55 ++++--------------- internal/machine/network.go | 26 +++++++-- internal/machine/network_darwin.go | 13 ----- 6 files changed, 101 insertions(+), 65 deletions(-) create mode 100644 internal/machine/docker.go create mode 100644 internal/machine/docker_darwin.go rename internal/machine/{network_linux.go => docker_linux.go} (59%) delete mode 100644 internal/machine/network_darwin.go diff --git a/internal/machine/api/proxy/local.go b/internal/machine/api/proxy/local.go index 38374d2b..805de6e5 100644 --- a/internal/machine/api/proxy/local.go +++ b/internal/machine/api/proxy/local.go @@ -2,7 +2,6 @@ package proxy import ( "context" - "fmt" "github.com/siderolabs/grpc-proxy/proxy" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -34,8 +33,6 @@ func (b *LocalBackend) String() string { // GetConnection returns a gRPC connection to the local server listening on the Unix socket. func (b *LocalBackend) GetConnection(ctx context.Context, _ string) (context.Context, *grpc.ClientConn, error) { md, _ := metadata.FromIncomingContext(ctx) - // TODO: delete - fmt.Printf("### local backend metadata: %+v\n", md) outCtx := metadata.NewOutgoingContext(ctx, md) b.mu.RLock() diff --git a/internal/machine/docker.go b/internal/machine/docker.go new file mode 100644 index 00000000..73611384 --- /dev/null +++ b/internal/machine/docker.go @@ -0,0 +1,55 @@ +package machine + +import ( + "context" + "fmt" + "github.com/docker/docker/client" + "log/slog" + "time" + "uncloud/internal/machine/store" +) + +const ( + DockerNetworkName = "uncloud" + DockerUserChain = "DOCKER-USER" +) + +type DockerManager struct { + client *client.Client + store *store.Store +} + +func NewDockerManager(client *client.Client, store *store.Store) *DockerManager { + return &DockerManager{ + client: client, + store: store, + } +} + +// WaitDaemonReady waits for the Docker daemon to start and be ready to serve requests. +func (d *DockerManager) WaitDaemonReady(ctx context.Context) error { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + ready, waitingLogged := false, false + for !ready { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + _, err := d.client.Ping(ctx) + if err == nil { + ready = true + break + } + if !client.IsErrConnectionFailed(err) { + return fmt.Errorf("connect to Docker daemon: %w", err) + } + if !waitingLogged { + slog.Info("Waiting for Docker daemon to start and be ready.") + waitingLogged = true + } + } + } + return nil +} diff --git a/internal/machine/docker_darwin.go b/internal/machine/docker_darwin.go new file mode 100644 index 00000000..5a5a29cd --- /dev/null +++ b/internal/machine/docker_darwin.go @@ -0,0 +1,14 @@ +//go:build darwin + +package machine + +import ( + "context" + "fmt" + "net/netip" +) + +// EnsureUncloudNetwork is a stub for darwin. +func (d *DockerManager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix) error { + return fmt.Errorf("not supported on darwin") +} diff --git a/internal/machine/network_linux.go b/internal/machine/docker_linux.go similarity index 59% rename from internal/machine/network_linux.go rename to internal/machine/docker_linux.go index 71e8f591..3ff3b3f9 100644 --- a/internal/machine/network_linux.go +++ b/internal/machine/docker_linux.go @@ -1,5 +1,3 @@ -//go:build linux - package machine import ( @@ -9,58 +7,29 @@ import ( "github.com/docker/docker/client" "github.com/docker/docker/libnetwork/iptables" "log/slog" - "time" + "net/netip" "uncloud/internal/machine/network" ) -// setupDockerNetwork creates the Docker bridge network DockerNetworkName with the machine subnet and configures -// iptables to allow WireGuard network to access containers. -func (nc *networkController) setupDockerNetwork(ctx context.Context) error { - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - return fmt.Errorf("init Docker client: %w", err) - } - defer cli.Close() - - // Wait for the Docker daemon to start and be ready by sending a ping request in a loop. - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - ready, waitingLogged := false, false - for !ready { - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - _, err = cli.Ping(ctx) - if err == nil { - ready = true - break - } - if !client.IsErrConnectionFailed(err) { - return fmt.Errorf("connect to Docker daemon: %w", err) - } - if !waitingLogged { - slog.Info("Waiting for Docker daemon to start and be ready to setup Docker network.") - waitingLogged = true - } - } - } - +// EnsureUncloudNetwork creates the Docker bridge network DockerNetworkName with the provided machine subnet +// if it doesn't exist. If the network exists but has a different subnet, it removes and recreates the network. +// It also configures iptables to allow container access from the WireGuard network. +func (d *DockerManager) EnsureUncloudNetwork(ctx context.Context, subnet netip.Prefix) error { // Ensure the Docker network 'uncloud' is created with the correct subnet. needsCreation := false - nw, err := cli.NetworkInspect(ctx, DockerNetworkName, dnetwork.InspectOptions{}) + nw, err := d.client.NetworkInspect(ctx, DockerNetworkName, dnetwork.InspectOptions{}) if err != nil { if !client.IsErrNotFound(err) { return fmt.Errorf("inspect Docker network %q: %w", DockerNetworkName, err) } needsCreation = true - } else if nw.IPAM.Config[0].Subnet != nc.state.Network.Subnet.String() { + } else if nw.IPAM.Config[0].Subnet != subnet.String() { // Remove the Docker network if the subnet is different. // It could be a leftover from a previous incomplete cleanup. slog.Info( "Removing Docker network with old subnet.", "name", DockerNetworkName, "subnet", nw.IPAM.Config[0].Subnet, ) - if err = cli.NetworkRemove(ctx, DockerNetworkName); err != nil { + if err = d.client.NetworkRemove(ctx, DockerNetworkName); err != nil { // It can still fail if the network is in use by a container. Leave it to the user to resolve the issue. return fmt.Errorf("remove Docker network %q: %w", DockerNetworkName, err) } @@ -68,14 +37,14 @@ func (nc *networkController) setupDockerNetwork(ctx context.Context) error { } if needsCreation { - if _, err = cli.NetworkCreate( + if _, err = d.client.NetworkCreate( ctx, DockerNetworkName, dnetwork.CreateOptions{ Driver: "bridge", Scope: "local", IPAM: &dnetwork.IPAM{ Config: []dnetwork.IPAMConfig{ { - Subnet: nc.state.Network.Subnet.String(), + Subnet: subnet.String(), }, }, }, @@ -83,9 +52,9 @@ func (nc *networkController) setupDockerNetwork(ctx context.Context) error { ); err != nil { return fmt.Errorf("create Docker network %q: %w", DockerNetworkName, err) } - slog.Info("Docker network created.", "name", DockerNetworkName, "subnet", nc.state.Network.Subnet.String()) + slog.Info("Docker network created.", "name", DockerNetworkName, "subnet", subnet.String()) - if nw, err = cli.NetworkInspect(ctx, DockerNetworkName, dnetwork.InspectOptions{}); err != nil { + if nw, err = d.client.NetworkInspect(ctx, DockerNetworkName, dnetwork.InspectOptions{}); err != nil { return fmt.Errorf("inspect Docker network %q: %w", DockerNetworkName, err) } } diff --git a/internal/machine/network.go b/internal/machine/network.go index 626284d6..3271bec6 100644 --- a/internal/machine/network.go +++ b/internal/machine/network.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "github.com/cenkalti/backoff/v4" + "github.com/docker/docker/client" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "log/slog" @@ -20,9 +21,7 @@ import ( ) const ( - APIPort = 51000 - DockerNetworkName = "uncloud" - DockerUserChain = "DOCKER-USER" + APIPort = 51000 ) type networkController struct { @@ -78,7 +77,7 @@ func (nc *networkController) Run(ctx context.Context) error { } } // TODO: Figure out if we need to manually stop the corrosion service when the context is done or just - // rely on systemd to handle service dependencies on its own . + // rely on systemd to handle service dependencies on its own. errGroup, ctx := errgroup.WithContext(ctx) @@ -101,10 +100,25 @@ func (nc *networkController) Run(ctx context.Context) error { // Setup Docker network and iptables rules in a goroutine because it may block until the Docker daemon is ready. errGroup.Go( func() error { - if err := nc.setupDockerNetwork(ctx); err != nil { - return fmt.Errorf("setup Docker network: %w", err) + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return fmt.Errorf("init Docker client: %w", err) + } + defer cli.Close() + + d := NewDockerManager(cli, nc.store) + if err := d.WaitDaemonReady(ctx); err != nil { + return fmt.Errorf("wait for Docker daemon: %w", err) + } + + if err := d.EnsureUncloudNetwork(ctx, nc.state.Network.Subnet); err != nil { + return err } slog.Info("Docker network configured.") + + //if err := d.WatchAndSyncContainers(ctx); err != nil { + // return fmt.Errorf("watch and sync containers to store: %w", err) + //} return nil }, ) diff --git a/internal/machine/network_darwin.go b/internal/machine/network_darwin.go deleted file mode 100644 index 2c40fdde..00000000 --- a/internal/machine/network_darwin.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build darwin - -package machine - -import ( - "context" - "fmt" -) - -// setupDockerNetwork is a stub for darwin. -func (nc *networkController) setupDockerNetwork(ctx context.Context) error { - return fmt.Errorf("not supported on darwin") -}