From f8234916cb3910b375b5de6d88120e0bce34f8af Mon Sep 17 00:00:00 2001 From: Pasha Sviderski Date: Wed, 15 Apr 2026 20:40:24 +1000 Subject: [PATCH] fix: another attempt to fix flaky e2e tests: bump dind and get rid of incomplete cgroups fix, pre-allocate ucind machine port, allow ucind container retarts --- Dockerfile | 4 +- internal/ucind/machine.go | 30 ++++++++++++- scripts/docker/dind | 83 ------------------------------------ scripts/docker/entrypoint.sh | 4 ++ 4 files changed, 34 insertions(+), 87 deletions(-) delete mode 100755 scripts/docker/dind diff --git a/Dockerfile b/Dockerfile index 88fbae3e..0707f7ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ RUN apk --no-cache add crane RUN crane pull --platform ${TARGETOS}/${TARGETARCH} "${CORROSION_IMAGE}" /corrosion.tar # Uncloud-in-Docker (ucind) image for running Uncloud test clusters using Docker. -FROM docker:27.3.1-dind AS ucind +FROM docker:29.4.0-dind AS ucind # Create system group and user 'uncloud'. RUN addgroup -S uncloud && adduser -SHD -h /nonexistent -G uncloud -g "" uncloud RUN apk --no-cache add \ @@ -54,7 +54,7 @@ RUN apk --no-cache add \ wireguard-tools COPY --from=corrosion-image-tarball /corrosion.tar /images/corrosion.tar -COPY scripts/docker/dind scripts/docker/entrypoint.sh /usr/local/bin/ +COPY scripts/docker/entrypoint.sh /usr/local/bin/ COPY --from=uncloudd /build/uncloudd /usr/local/bin/ ENTRYPOINT ["entrypoint.sh"] diff --git a/internal/ucind/machine.go b/internal/ucind/machine.go index 711b9dcb..3246e4d1 100644 --- a/internal/ucind/machine.go +++ b/internal/ucind/machine.go @@ -7,6 +7,7 @@ import ( "io" "net" "net/netip" + "strconv" "time" "github.com/containerd/errdefs" @@ -62,6 +63,16 @@ func (p *Provisioner) CreateMachine(ctx context.Context, clusterName string, opt } apiPort := nat.Port(fmt.Sprintf("%d/tcp", UncloudAPIPort)) + // Pre-allocate a free host port on 127.0.0.1 and bind it explicitly. Letting Docker assign a random port + // (HostPort: "") breaks the cached machine address when the container restarts because Docker picks a new random + // port each time. + // It's been noticed ucind restarts on rare first-boot failures, which showed up as flaky + // "connection refused" errors in e2e tests. This has been addressed by upgrading to the latest dind image. + hostPort, err := availableLocalPort() + if err != nil { + return m, fmt.Errorf("reserve host port for machine API: %w", err) + } + config := &container.Config{ Image: img, Labels: map[string]string{ @@ -78,8 +89,8 @@ func (p *Provisioner) CreateMachine(ctx context.Context, clusterName string, opt PortBindings: nat.PortMap{ apiPort: []nat.PortBinding{ { - HostIP: "127.0.0.1", - // Host port is a random available port. + HostIP: "127.0.0.1", + HostPort: strconv.Itoa(hostPort), }, }, }, @@ -185,6 +196,21 @@ func (p *Provisioner) waitPortPublished(ctx context.Context, containerID string, } } +// availableLocalPort asks the kernel for a free TCP port on 127.0.0.1 and returns it. +func availableLocalPort() (int, error) { + l, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}) + if err != nil { + return 0, err + } + + port := l.Addr().(*net.TCPAddr).Port + if err = l.Close(); err != nil { + return 0, err + } + + return port, nil +} + func randomMachineName() (string, error) { suffix, err := secret.RandomAlphaNumeric(4) if err != nil { diff --git a/scripts/docker/dind b/scripts/docker/dind deleted file mode 100755 index d552d36c..00000000 --- a/scripts/docker/dind +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/sh -# This is a fork of https://github.com/moby/moby/blob/65cfcc28ab37cb75e1560e4b4738719c07c6618e/hack/dind -# with a fix for cgroup v2 initialization races. - -set -e - -# DinD: a wrapper script which allows docker to be run inside a docker container. -# Original version by Jerome Petazzoni -# See the blog post: https://www.docker.com/blog/docker-can-now-run-within-docker/ -# -# This script should be executed inside a docker container in privileged mode -# ('docker run --privileged', introduced in docker 0.6). - -# Usage: dind CMD [ARG...] - -# apparmor sucks and Docker needs to know that it's in a container (c) @tianon -# -# Set the container env-var, so that AppArmor is enabled in the daemon and -# containerd when running docker-in-docker. -# -# see: https://github.com/containerd/containerd/blob/787943dc1027a67f3b52631e084db0d4a6be2ccc/pkg/apparmor/apparmor_linux.go#L29-L45 -# see: https://github.com/moby/moby/commit/de191e86321f7d3136ff42ff75826b8107399497 -export container=docker - -# Allow AppArmor to work inside the container; -# -# aa-status -# apparmor filesystem is not mounted. -# apparmor module is loaded. -# -# mount -t securityfs none /sys/kernel/security -# -# aa-status -# apparmor module is loaded. -# 30 profiles are loaded. -# 30 profiles are in enforce mode. -# /snap/snapd/18357/usr/lib/snapd/snap-confine -# ... -# -# Note: https://0xn3va.gitbook.io/cheat-sheets/container/escaping/sensitive-mounts#sys-kernel-security -# -# ## /sys/kernel/security -# -# In /sys/kernel/security mounted the securityfs interface, which allows -# configuration of Linux Security Modules. This allows configuration of -# AppArmor policies, and so access to this may allow a container to disable -# its MAC system. -# -# Given that we're running privileged already, this should not be an issue. -if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security; then - mount -t securityfs none /sys/kernel/security || { - echo >&2 'Could not mount /sys/kernel/security.' - echo >&2 'AppArmor detection and --privileged mode might break.' - } -fi - -# Mount /tmp (conditionally) -if ! mountpoint -q /tmp; then - mount -t tmpfs none /tmp -fi - -# cgroup v2: enable nesting -if [ -f /sys/fs/cgroup/cgroup.controllers ]; then - # move the processes from the root group to the /init group, - # otherwise writing subtree_control fails with EBUSY. - # An error during moving non-existent process (i.e., "cat") is ignored. - mkdir -p /sys/fs/cgroup/init - xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : - # enable controllers with retry on cgroup initialization races - timeout 5s sh -c "until sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ - > /sys/fs/cgroup/cgroup.subtree_control 2>/dev/null; do sleep 0.1; done" -fi - -# Change mount propagation to shared to make the environment more similar to a -# modern Linux system, e.g. with SystemD as PID 1. -mount --make-rshared / - -if [ $# -gt 0 ]; then - exec "$@" -fi - -echo >&2 'ERROR: No command specified.' -echo >&2 'You probably want to run hack/make.sh, or maybe a shell?' diff --git a/scripts/docker/entrypoint.sh b/scripts/docker/entrypoint.sh index 5f5ad7c9..d67309b3 100755 --- a/scripts/docker/entrypoint.sh +++ b/scripts/docker/entrypoint.sh @@ -23,6 +23,10 @@ cleanup() { } trap cleanup INT TERM EXIT +# Remove stale Docker and containerd pid files left over from the previous container run. They persist +# across restarts because /run lives in the container's overlay filesystem, not a tmpfs. +rm -f /run/docker.pid /run/docker/containerd/containerd.pid + dind dockerd & echo "Waiting for Docker in Docker to be ready..." timeout 60s sh -c "until docker info &> /dev/null; do sleep 0.5; done"