feat: support rootless docker when pushing images to cluster (#347)

pkg/client/image.go

@@ -5,7 +5,10 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log/slog"
 	"net"
+	"os"
+	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@@ -32,6 +35,12 @@ import (
 	netproxy "golang.org/x/net/proxy"
 )
 
+// This is the container image used to run socat proxy containers.
+// TODO: it's an external dependency that we don't control, so we should consider vendoring it or
+// automatically building an ad-hoc image with socat-like functionality (e.g. using a lightweight Go
+// proxy) as part of the push process.
+const socatImage = "alpine/socat:1.8.0.3"
+
 func (cli *Client) InspectImage(ctx context.Context, id string) ([]api.MachineImage, error) {
 	images, err := cli.Docker.InspectImage(ctx, id)
 	if errdefs.IsNotFound(err) {
@@ -235,26 +244,55 @@ func (cli *Client) pushImageToMachine(
 		return fmt.Errorf("get proxy dialer: %w", err)
 	}
 
+	dockerEnv, err := detectDockerEnvironment(ctx, dockerCli)
+	if err != nil {
+		return err
+	}
+	slog.Debug("Detected Docker environment:", "virtualised", dockerEnv.Virtualised, "rootless", dockerEnv.Rootless)
+
 	proxyEventID := fmt.Sprintf("Proxy to unregistry on %s", boldStyle.Render(machine.Name))
 	pw.Event(progress.StartingEvent(proxyEventID))
 
 	// The proxy runs in a goroutine. Capture the first error in a channel
 	// so we can surface it alongside the push error if push fails.
 	proxyErrCh := make(chan error, 1)
-	// Forward local port 127.0.0.1:PORT to the machine's unregistry over the established client connection.
-	unregProxy, err := newUnregistryProxy(ctx, unregistryAddr, dialer, func(err error) {
+	onProxyError := func(err error) {
 		select {
 		case proxyErrCh <- fmt.Errorf("proxy to unregistry: %w", err):
 		default:
 		}
 		pw.Event(progress.NewEvent(proxyEventID, progress.Error, err.Error()))
-	})
+	}
+
+	// socketPath is set for plain rootless Docker (not running inside a VM): the Go proxy listens on a unix
+	// socket that is bind-mounted into the socat container, bypassing slirp4netns network routing entirely.
+	var (
+		socketPath string
+		proxyPort  int
+	)
+	var unregProxy *proxy.Proxy
+	if shouldUseUnregistryUnixProxy(dockerEnv) {
+		suffix, err := secret.RandomAlphaNumeric(4)
 		if err != nil {
 			pw.Event(progress.NewEvent(proxyEventID, progress.Error, err.Error()))
-		return fmt.Errorf("create local proxy to unregistry on machine '%s': %w", machine.Name, err)
+			return fmt.Errorf("generate socket path suffix: %w", err)
+		}
+		socketPath = filepath.Join(os.TempDir(), fmt.Sprintf("uncloud-push-%s.sock", suffix))
+		unregProxy, err = newUnregistryUnixProxy(ctx, unregistryAddr, dialer, socketPath, onProxyError)
+		if err != nil {
+			pw.Event(progress.NewEvent(proxyEventID, progress.Error, err.Error()))
+			return fmt.Errorf("create local unix socket proxy to unregistry on machine '%s': %w", machine.Name, err)
+		}
+		slog.Debug("Listening for unregistry proxy connections.", "mode", "unix", "socket", socketPath)
+	} else {
+		unregProxy, err = newUnregistryTcpProxy(ctx, unregistryAddr, dialer, onProxyError)
+		if err != nil {
+			pw.Event(progress.NewEvent(proxyEventID, progress.Error, err.Error()))
+			return fmt.Errorf("create local tcp proxy to unregistry on machine '%s': %w", machine.Name, err)
+		}
+		proxyPort = unregProxy.Listener.Addr().(*net.TCPAddr).Port
+		slog.Debug("Listening for unregistry proxy connections.", "mode", "tcp", "port", proxyPort)
 	}
-	// Get the local port the unregistry proxy is listening on.
-	proxyPort := unregProxy.Listener.Addr().(*net.TCPAddr).Port
 
 	proxyCtx, cancelProxy := context.WithCancel(ctx)
 	proxyCtrID := ""
@@ -272,31 +310,47 @@ func (cli *Client) pushImageToMachine(
 			dockerCli.ContainerRemove(ctx, proxyCtrID, container.RemoveOptions{Force: true})
 		}
 
+		// Remove the unix socket file used for rootless Docker proxying.
+		if socketPath != "" {
+			os.Remove(socketPath)
+		}
+
 		cancelProxy()
 	}
 	defer cleanup()
 
 	go unregProxy.Run(proxyCtx)
 
-	dockerVirtualised, err := isDockerVirtualised(ctx, dockerCli)
-	if err != nil {
-		return err
-	}
-
-	if dockerVirtualised {
-		// Run socat proxy container to forward a localhost port from within the Docker VM to the host machine.
+	if dockerEnv.Virtualised {
+		// VM-based Docker (Docker Desktop, Rancher Desktop, etc.): run a socat container inside the VM
+		// to forward a port via host.docker.internal back to the host-side proxy.
 		pw.Event(progress.Event{
 			ID:         proxyEventID,
 			Status:     progress.Working,
 			StatusText: "Starting",
 			Text:       "(detected Docker in VM locally, starting socat proxy container)",
 		})
-
 		proxyCtrID, proxyPort, err = runDockerVMProxyContainer(ctx, dockerCli, proxyPort)
 		if err != nil {
 			pw.Event(progress.NewEvent(proxyEventID, progress.Error, err.Error()))
-			return fmt.Errorf("run socat container to proxy unregistry to Docker VM: %w", err)
+			return fmt.Errorf("run socat container to proxy unregistry: %w", err)
 		}
+		slog.Debug("Started VM socat proxy container.", "id", proxyCtrID, "hostPort", proxyPort)
+	} else if shouldUseUnregistryUnixProxy(dockerEnv) {
+		// Plain rootless Docker: run a socat container that forwards via a bind-mounted unix socket,
+		// bypassing the slirp4netns --disable-host-loopback restriction.
+		pw.Event(progress.Event{
+			ID:         proxyEventID,
+			Status:     progress.Working,
+			StatusText: "Starting",
+			Text:       "(detected rootless Docker, starting socat proxy container)",
+		})
+		proxyCtrID, proxyPort, err = runUnixSocketProxyContainer(ctx, dockerCli, socketPath)
+		if err != nil {
+			pw.Event(progress.NewEvent(proxyEventID, progress.Error, err.Error()))
+			return fmt.Errorf("run socat container with unix socket to proxy unregistry: %w", err)
+		}
+		slog.Debug("Started unix socket socat proxy container.", "id", proxyCtrID, "hostPort", proxyPort, "socket", socketPath)
 	}
 
 	pw.Event(progress.Event{
@@ -350,18 +404,27 @@ func (cli *Client) pushImageToMachine(
 	return nil
 }
 
-func newUnregistryProxy(
-	ctx context.Context, remoteAddr string, dialer netproxy.ContextDialer, onError func(error),
-) (*proxy.Proxy, error) {
-	// Test remote connectivity before creating a proxy.
+// checkRemoteConnectivity verifies that remoteAddr is reachable via dialer within a timeout.
+func checkRemoteConnectivity(ctx context.Context, dialer netproxy.ContextDialer, remoteAddr string) error {
 	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
 
-	testConn, err := dialer.DialContext(ctx, "tcp", remoteAddr)
+	conn, err := dialer.DialContext(ctx, "tcp", remoteAddr)
 	if err != nil {
-		return nil, fmt.Errorf("connect to remote address '%s': %w", remoteAddr, err)
+		return fmt.Errorf("connect to remote address '%s': %w", remoteAddr, err)
+	}
+	conn.Close()
+	return nil
+}
+
+// newUnregistryTcpProxy creates a TCP proxy that listens on localhost and forwards to the unregistry
+// address on the target machine.
+func newUnregistryTcpProxy(
+	ctx context.Context, remoteAddr string, dialer netproxy.ContextDialer, onError func(error),
+) (*proxy.Proxy, error) {
+	if err := checkRemoteConnectivity(ctx, dialer, remoteAddr); err != nil {
+		return nil, err
 	}
-	testConn.Close()
 
 	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
@@ -378,39 +441,109 @@ func newUnregistryProxy(
 	return p, nil
 }
 
-// isDockerVirtualised checks if Docker is running in a virtualised environment like Docker/Rancher Desktop or Colima.
-// On macOS, Docker always requires a VM, so it returns true unless OrbStack is detected (which handles host networking
-// natively). On other platforms, it checks for known virtualised Docker hostnames.
-func isDockerVirtualised(ctx context.Context, dockerCli *docker.Client) (bool, error) {
-	info, err := dockerCli.Info(ctx)
-	if err != nil {
-		return false, fmt.Errorf("get Docker info: %w", err)
+// newUnregistryUnixProxy creates a proxy that listens on a unix socket at socketPath and forwards to the
+// unregistry address on the target machine.
+func newUnregistryUnixProxy(
+	ctx context.Context, remoteAddr string, dialer netproxy.ContextDialer, socketPath string, onError func(error),
+) (*proxy.Proxy, error) {
+	if err := checkRemoteConnectivity(ctx, dialer, remoteAddr); err != nil {
+		return nil, err
 	}
 
-	// On macOS, Docker always runs in a VM. OrbStack is the only known exception that doesn't need a proxy.
-	if runtime.GOOS == "darwin" {
-		if info.Name == "orbstack" {
-			return false, nil
+	listener, err := net.Listen("unix", socketPath)
+	if err != nil {
+		return nil, fmt.Errorf("listen on unix socket %s: %w", socketPath, err)
 	}
-		return true, nil
+	// Restrict socket access to the current user to prevent other local users from connecting
+	// to the proxy and reaching the unregistry for the duration of the push.
+	if err = os.Chmod(socketPath, 0o600); err != nil {
+		listener.Close()
+		return nil, fmt.Errorf("set unix socket permissions on %s: %w", socketPath, err)
+	}
+
+	return &proxy.Proxy{
+		Listener:    listener,
+		RemoteAddr:  remoteAddr,
+		DialContext: dialer.DialContext,
+		OnError:     onError,
+	}, nil
+}
+
+// dockerEnvironment describes the local Docker environment type.
+type dockerEnvironment struct {
+	// Virtualised indicates Docker runs inside a VM (Docker Desktop, Rancher Desktop, Colima).
+	Virtualised bool
+	// Rootless indicates Docker is running in rootless mode
+	Rootless bool
+}
+
+// detectDockerEnvironment inspects the local Docker daemon to determine if it is running in a virtualised
+// environment (VM) or in rootless mode. Both cases require a socat proxy container to route connections
+// across the network boundary between the local Docker daemon and the host.
+func detectDockerEnvironment(ctx context.Context, dockerCli *docker.Client) (dockerEnvironment, error) {
+	info, err := dockerCli.Info(ctx)
+	if err != nil {
+		return dockerEnvironment{}, fmt.Errorf("get Docker info: %w", err)
+	}
+
+	env := dockerEnvironment{}
+
+	for _, opt := range info.SecurityOptions {
+		if strings.Contains(opt, "rootless") {
+			env.Rootless = true
+			break
+		}
+	}
+
+	// On macOS, Docker always requires a VM, so set Virtualised to true unless OrbStack is
+	// detected (which handles host networking natively).
+	if runtime.GOOS == "darwin" {
+		if info.Name != "orbstack" {
+			env.Virtualised = true
+		}
+		return env, nil
 	}
 
 	// On other platforms, check for known virtualised Docker environments.
 	virtualisedHostnames := []string{"docker-desktop", "rancher-desktop", "colima"}
 	for _, name := range virtualisedHostnames {
 		if strings.Contains(strings.ToLower(info.Name), name) {
-			return true, nil
+			env.Virtualised = true
+			break
 		}
 	}
+	return env, nil
+}
 
-	return false, nil
+// shouldUseUnregistryUnixProxy reports whether the local Docker environment requires a unix socket proxy
+// to reach the unregistry. This is the case for rootless Docker not running inside a VM, where
+// slirp4netns --disable-host-loopback blocks TCP routing from the container network namespace back to the host.
+func shouldUseUnregistryUnixProxy(env dockerEnvironment) bool {
+	// TODO: handle Virtualised AND Rootless case when we encounter it.
+	return env.Rootless && !env.Virtualised
 }
 
-// runDockerVMProxyContainer creates a socat container to proxy an available localhost port within the Docker VM
-// (e.g. Docker Desktop on macOS) to the specified target port on the host machine.
+// runDockerVMProxyContainer creates a socat container inside the Docker VM (e.g. Docker Desktop on macOS)
+// to forward TCP connections from a localhost port to the specified target port on the host via host.docker.internal.
 // Returns the container ID and the localhost port the container port is bound to.
-// TODO: accept custom image name.
 func runDockerVMProxyContainer(ctx context.Context, dockerCli *docker.Client, targetPort int) (string, int, error) {
+	return runSocatProxyContainer(ctx, dockerCli,
+		fmt.Sprintf("TCP-CONNECT:host.docker.internal:%d", targetPort), nil)
+}
+
+// runUnixSocketProxyContainer creates a socat container that forwards TCP connections to the host-side Go
+// proxy via a bind-mounted unix socket.
+// Returns the container ID and the localhost port that 'docker push' should target.
+func runUnixSocketProxyContainer(ctx context.Context, dockerCli *docker.Client, socketPath string) (string, int, error) {
+	return runSocatProxyContainer(ctx, dockerCli,
+		fmt.Sprintf("UNIX-CONNECT:%s", socketPath),
+		[]string{fmt.Sprintf("%s:%s", socketPath, socketPath)})
+}
+
+// runSocatProxyContainer creates a socat container that listens on TCP port 5000 and forwards to socatDst.
+// binds is an optional list of host:container bind mounts (e.g. for a unix socket).
+// Returns the container ID and the localhost port that clients should connect to.
+func runSocatProxyContainer(ctx context.Context, dockerCli *docker.Client, socatDst string, binds []string) (string, int, error) {
 	suffix, err := secret.RandomAlphaNumeric(4)
 	if err != nil {
 		return "", 0, fmt.Errorf("generate random suffix: %w", err)
@@ -420,14 +553,14 @@ func runDockerVMProxyContainer(ctx context.Context, dockerCli *docker.Client, ta
 	containerPort := nat.Port("5000/tcp")
 	config := &container.Config{
 		// TODO: make image configurable.
-		Image: "alpine/socat:latest",
+		Image: socatImage,
 		// Reset the default entrypoint "socat".
 		Entrypoint: []string{},
 		Cmd: []string{
 			"timeout", "1800", // Auto-terminate socat after 30 minutes.
 			"socat",
 			"TCP-LISTEN:5000,fork,reuseaddr",
-			fmt.Sprintf("TCP-CONNECT:host.docker.internal:%d", targetPort),
+			socatDst,
 		},
 		ExposedPorts: nat.PortSet{
 			containerPort: {},
@@ -449,6 +582,7 @@ func runDockerVMProxyContainer(ctx context.Context, dockerCli *docker.Client, ta
 
 	hostConfig := &container.HostConfig{
 		AutoRemove: true,
+		Binds:      binds,
 		PortBindings: nat.PortMap{
 			containerPort: []nat.PortBinding{
 				{
@@ -474,9 +608,32 @@ func runDockerVMProxyContainer(ctx context.Context, dockerCli *docker.Client, ta
 		return "", 0, fmt.Errorf("start socat proxy container %s: %w", resp.ID, err)
 	}
 
+	// Wait for socat to start listening inside the container. ContainerStart returns as soon as the
+	// container process is launched, but socat may not have bound its port yet. Without this check,
+	// the first 'docker push' connection can arrive before socat is ready and get reset.
+	addr := fmt.Sprintf("127.0.0.1:%d", hostPort)
+	if err = waitForTCPPort(addr, 10*time.Second); err != nil {
+		cleanup()
+		return "", 0, fmt.Errorf("socat proxy container %s: %w", resp.ID, err)
+	}
+
 	return resp.ID, hostPort, nil
 }
 
+// waitForTCPPort polls addr until a TCP connection succeeds or timeout is reached.
+func waitForTCPPort(addr string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, time.Second)
+		if err == nil {
+			conn.Close()
+			return nil
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return fmt.Errorf("port %s did not become ready within %s", addr, timeout)
+}
+
 // toPushProgressEvent converts a JSON progress message from the Docker API to a progress event.
 // It's based on toPushProgressEvent from Docker Compose.
 func toPushProgressEvent(jm jsonmessage.JSONMessage) *progress.Event {