fix: another attempt to fix flaky e2e tests: bump dind and get rid of incomplete cgroups fix, pre-allocate ucind machine port, allow ucind container retarts

Dockerfile

@@ -46,7 +46,7 @@ RUN apk --no-cache add crane
 RUN crane pull --platform ${TARGETOS}/${TARGETARCH} "${CORROSION_IMAGE}" /corrosion.tar
 
 # Uncloud-in-Docker (ucind) image for running Uncloud test clusters using Docker.
-FROM docker:27.3.1-dind AS ucind
+FROM docker:29.4.0-dind AS ucind
 # Create system group and user 'uncloud'.
 RUN addgroup -S uncloud && adduser -SHD -h /nonexistent -G uncloud -g "" uncloud
 RUN apk --no-cache add \
@@ -54,7 +54,7 @@ RUN apk --no-cache add \
     wireguard-tools
 
 COPY --from=corrosion-image-tarball /corrosion.tar /images/corrosion.tar
-COPY scripts/docker/dind scripts/docker/entrypoint.sh /usr/local/bin/
+COPY scripts/docker/entrypoint.sh /usr/local/bin/
 COPY --from=uncloudd /build/uncloudd /usr/local/bin/
 
 ENTRYPOINT ["entrypoint.sh"]

internal/ucind/machine.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net"
 	"net/netip"
+	"strconv"
 	"time"
 
 	"github.com/containerd/errdefs"
@@ -62,6 +63,16 @@ func (p *Provisioner) CreateMachine(ctx context.Context, clusterName string, opt
 	}
 
 	apiPort := nat.Port(fmt.Sprintf("%d/tcp", UncloudAPIPort))
+	// Pre-allocate a free host port on 127.0.0.1 and bind it explicitly. Letting Docker assign a random port
+	// (HostPort: "") breaks the cached machine address when the container restarts because Docker picks a new random
+	// port each time.
+	// It's been noticed ucind restarts on rare first-boot failures, which showed up as flaky
+	// "connection refused" errors in e2e tests. This has been addressed by upgrading to the latest dind image.
+	hostPort, err := availableLocalPort()
+	if err != nil {
+		return m, fmt.Errorf("reserve host port for machine API: %w", err)
+	}
+
 	config := &container.Config{
 		Image: img,
 		Labels: map[string]string{
@@ -79,7 +90,7 @@ func (p *Provisioner) CreateMachine(ctx context.Context, clusterName string, opt
 			apiPort: []nat.PortBinding{
 				{
 					HostIP:   "127.0.0.1",
-					// Host port is a random available port.
+					HostPort: strconv.Itoa(hostPort),
 				},
 			},
 		},
@@ -185,6 +196,21 @@ func (p *Provisioner) waitPortPublished(ctx context.Context, containerID string,
 	}
 }
 
+// availableLocalPort asks the kernel for a free TCP port on 127.0.0.1 and returns it.
+func availableLocalPort() (int, error) {
+	l, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
+	if err != nil {
+		return 0, err
+	}
+
+	port := l.Addr().(*net.TCPAddr).Port
+	if err = l.Close(); err != nil {
+		return 0, err
+	}
+
+	return port, nil
+}
+
 func randomMachineName() (string, error) {
 	suffix, err := secret.RandomAlphaNumeric(4)
 	if err != nil {

scripts/docker/dind

@@ -1,83 +0,0 @@
-#!/bin/sh
-# This is a fork of https://github.com/moby/moby/blob/65cfcc28ab37cb75e1560e4b4738719c07c6618e/hack/dind
-# with a fix for cgroup v2 initialization races.
-
-set -e
-
-# DinD: a wrapper script which allows docker to be run inside a docker container.
-# Original version by Jerome Petazzoni <jerome@docker.com>
-# See the blog post: https://www.docker.com/blog/docker-can-now-run-within-docker/
-#
-# This script should be executed inside a docker container in privileged mode
-# ('docker run --privileged', introduced in docker 0.6).
-
-# Usage: dind CMD [ARG...]
-
-# apparmor sucks and Docker needs to know that it's in a container (c) @tianon
-#
-# Set the container env-var, so that AppArmor is enabled in the daemon and
-# containerd when running docker-in-docker.
-#
-# see: https://github.com/containerd/containerd/blob/787943dc1027a67f3b52631e084db0d4a6be2ccc/pkg/apparmor/apparmor_linux.go#L29-L45
-# see: https://github.com/moby/moby/commit/de191e86321f7d3136ff42ff75826b8107399497
-export container=docker
-
-# Allow AppArmor to work inside the container;
-#
-#     aa-status
-#     apparmor filesystem is not mounted.
-#     apparmor module is loaded.
-#
-#     mount -t securityfs none /sys/kernel/security
-#
-#     aa-status
-#     apparmor module is loaded.
-#     30 profiles are loaded.
-#     30 profiles are in enforce mode.
-#       /snap/snapd/18357/usr/lib/snapd/snap-confine
-#       ...
-#
-# Note: https://0xn3va.gitbook.io/cheat-sheets/container/escaping/sensitive-mounts#sys-kernel-security
-#
-#     ## /sys/kernel/security
-#
-#     In /sys/kernel/security mounted the securityfs interface, which allows
-#     configuration of Linux Security Modules. This allows configuration of
-#     AppArmor policies, and so access to this may allow a container to disable
-#     its MAC system.
-#
-# Given that we're running privileged already, this should not be an issue.
-if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security; then
-	mount -t securityfs none /sys/kernel/security || {
-		echo >&2 'Could not mount /sys/kernel/security.'
-		echo >&2 'AppArmor detection and --privileged mode might break.'
-	}
-fi
-
-# Mount /tmp (conditionally)
-if ! mountpoint -q /tmp; then
-	mount -t tmpfs none /tmp
-fi
-
-# cgroup v2: enable nesting
-if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
-	# move the processes from the root group to the /init group,
-	# otherwise writing subtree_control fails with EBUSY.
-	# An error during moving non-existent process (i.e., "cat") is ignored.
-	mkdir -p /sys/fs/cgroup/init
-	xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
-	# enable controllers with retry on cgroup initialization races
-	timeout 5s sh -c "until sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
-		> /sys/fs/cgroup/cgroup.subtree_control 2>/dev/null; do sleep 0.1; done"
-fi
-
-# Change mount propagation to shared to make the environment more similar to a
-# modern Linux system, e.g. with SystemD as PID 1.
-mount --make-rshared /
-
-if [ $# -gt 0 ]; then
-	exec "$@"
-fi
-
-echo >&2 'ERROR: No command specified.'
-echo >&2 'You probably want to run hack/make.sh, or maybe a shell?'

scripts/docker/entrypoint.sh

@@ -23,6 +23,10 @@ cleanup() {
 }
 trap cleanup INT TERM EXIT
 
+# Remove stale Docker and containerd pid files left over from the previous container run. They persist
+# across restarts because /run lives in the container's overlay filesystem, not a tmpfs.
+rm -f /run/docker.pid /run/docker/containerd/containerd.pid
+
 dind dockerd &
 echo "Waiting for Docker in Docker to be ready..."
 timeout 60s sh -c "until docker info &> /dev/null; do sleep 0.5; done"