feat(distlock): integrate distributed lease management into Machine API

* feat: support stdin_open and tty

This can be useful to leave a container running without specifying a
command like `sleep`, and probably in other situations as well.

Signed-off-by: Miek Gieben <miek@miek.nl>

* Complete full spec test case

Signed-off-by: Miek Gieben <miek@miek.nl>

* fix container log streaming for containers with TTY

---------

Signed-off-by: Miek Gieben <miek@miek.nl>
Co-authored-by: Pasha Sviderski <me@psviderski.name>

cmd/uc/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/psviderski/uncloud/internal/machine"
 	"github.com/psviderski/uncloud/internal/version"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
 type globalOptions struct {
@@ -42,6 +43,13 @@ func main() {
 		SilenceUsage:  true,
 		SilenceErrors: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			// Shell completion runs through the hidden __complete command which has flag parsing disabled,
+			// so the global flags from the completed command line are never parsed. Apply them manually to make
+			// completion work with --connect, --context, and --uncloud-config.
+			if cmd.Name() == cobra.ShellCompRequestCmd {
+				applyGlobalFlagsFromCompletionArgs(cmd.Root().PersistentFlags(), os.Args[1:])
+			}
+
 			cli.BindEnvToFlag(cmd, "connect", "UNCLOUD_CONNECT")
 			cli.BindEnvToFlag(cmd, "context", "UNCLOUD_CONTEXT")
 			cli.BindEnvToFlag(cmd, "uncloud-config", "UNCLOUD_CONFIG")
@@ -159,3 +167,27 @@ func main() {
 		cobra.CheckErr(err)
 	}
 }
+
+// applyGlobalFlagsFromCompletionArgs parses the global flags from the raw arguments of a __complete command and applies
+// the ones found to flags. The trailing word being completed, unknown flags, and positional arguments are ignored.
+func applyGlobalFlagsFromCompletionArgs(flags *pflag.FlagSet, args []string) {
+	// The shell always passes the word being completed as the last argument, even if it's empty.
+	// Exclude it from parsing as its value may not be complete yet.
+	if len(args) == 0 {
+		return
+	}
+	args = args[:len(args)-1]
+
+	fset := pflag.NewFlagSet("global", pflag.ContinueOnError)
+	fset.ParseErrorsAllowlist.UnknownFlags = true
+	fset.String("connect", "", "")
+	fset.StringP("context", "c", "", "")
+	fset.String("uncloud-config", "", "")
+	// Parsing an incomplete command line may fail, apply the flags parsed so far anyway.
+	_ = fset.Parse(args)
+
+	fset.Visit(func(f *pflag.Flag) {
+		// Setting the flag marks it as changed so it takes precedence over environment variables.
+		_ = flags.Set(f.Name, f.Value.String())
+	})
+}

cmd/uc/main_test.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"slices"
+	"testing"
+
+	"github.com/spf13/pflag"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestApplyGlobalFlagsFromCompletionArgs(t *testing.T) {
+	defaultConfigPath := "~/.config/uncloud/config.yaml"
+
+	tests := []struct {
+		name           string
+		args           []string
+		wantConnect    string
+		wantContext    string
+		wantConfigPath string
+		// Flag names expected to be marked as changed on the target flag set.
+		wantChanged []string
+	}{
+		{
+			name: "no flags",
+			args: []string{"__complete", "inspect", ""},
+		},
+		{
+			name:        "connect with space",
+			args:        []string{"__complete", "--connect", "ssh://user@host", "inspect", ""},
+			wantConnect: "ssh://user@host",
+			wantChanged: []string{"connect"},
+		},
+		{
+			name:        "connect with equals",
+			args:        []string{"__complete", "--connect=tcp://127.0.0.1:51000", "inspect", ""},
+			wantConnect: "tcp://127.0.0.1:51000",
+			wantChanged: []string{"connect"},
+		},
+		{
+			name:        "context shorthand",
+			args:        []string{"__complete", "-c", "prod", "inspect", ""},
+			wantContext: "prod",
+			wantChanged: []string{"context"},
+		},
+		{
+			name:           "all flags",
+			args:           []string{"__complete", "--connect", "user@host", "-c", "prod", "--uncloud-config", "/tmp/uncloud.yaml", "inspect", ""},
+			wantConnect:    "user@host",
+			wantContext:    "prod",
+			wantConfigPath: "/tmp/uncloud.yaml",
+			wantChanged:    []string{"connect", "context", "uncloud-config"},
+		},
+		{
+			name:        "unknown flags are ignored",
+			args:        []string{"__complete", "--quiet", "-n", "5", "--connect", "user@host", "logs", ""},
+			wantConnect: "user@host",
+			wantChanged: []string{"connect"},
+		},
+		{
+			name: "flags after double dash are ignored",
+			args: []string{"__complete", "exec", "svc", "--", "sh", "--connect", "user@host"},
+		},
+		{
+			name:        "flags before double dash are applied",
+			args:        []string{"__complete", "--connect", "user@host", "exec", "svc", "--", "sh", "-c", "env"},
+			wantConnect: "user@host",
+			wantChanged: []string{"connect"},
+		},
+		{
+			name:        "partial flag name being completed is excluded",
+			args:        []string{"__complete", "--connect", "user@host", "inspect", "--context"},
+			wantConnect: "user@host",
+			wantChanged: []string{"connect"},
+		},
+		{
+			name: "partial flag value being completed is excluded",
+			args: []string{"__complete", "--uncloud-config", "/tmp/"},
+		},
+		{
+			name: "partial connect value being completed is excluded",
+			args: []string{"__complete", "--connect", "tcp://127.0.0.1:5"},
+		},
+		{
+			name:           "completed flag value with partial command word",
+			args:           []string{"__complete", "--uncloud-config", "/tmp/uncloud.yaml", "insp"},
+			wantConfigPath: "/tmp/uncloud.yaml",
+			wantChanged:    []string{"uncloud-config"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Mirror the global persistent flags defined on the root command.
+			var opts globalOptions
+			flags := pflag.NewFlagSet("test", pflag.ContinueOnError)
+			flags.StringVar(&opts.connect, "connect", "", "")
+			flags.StringVarP(&opts.context, "context", "c", "", "")
+			flags.StringVar(&opts.configPath, "uncloud-config", defaultConfigPath, "")
+
+			applyGlobalFlagsFromCompletionArgs(flags, tt.args)
+
+			assert.Equal(t, tt.wantConnect, opts.connect)
+			assert.Equal(t, tt.wantContext, opts.context)
+			wantConfigPath := tt.wantConfigPath
+			if wantConfigPath == "" {
+				wantConfigPath = defaultConfigPath
+			}
+			assert.Equal(t, wantConfigPath, opts.configPath)
+
+			for _, name := range []string{"connect", "context", "uncloud-config"} {
+				assert.Equal(t, slices.Contains(tt.wantChanged, name), flags.Changed(name),
+					"changed status of flag '%s'", name)
+			}
+		})
+	}
+}

cmd/uc/proxy.go

@@ -132,35 +132,34 @@ func runProxy(ctx context.Context, uncli *cli.CLI, opts proxyOptions) error {
 	// endpoint and shuffles the data, *it* will actually experience errors.
 	remoteAddr := net.JoinHostPort(ip.String(), strconv.Itoa(opts.remotePort))
 
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
 	p := &proxy.Proxy{
 		Listener:    listener,
 		RemoteAddr:  remoteAddr,
 		DialContext: dialer.DialContext,
 		OnError: func(err error) {
-			fmt.Printf("Failed to proxy to '%s': %v\n", remoteAddr, err)
-			cancel()
-		},
+			if proxy.IsConnectionClosedError(err) {
+				return
+			}
+			// A more actionable error instead of the cryptic [ssh -W] command error.
+			if strings.Contains(err.Error(), "Session open refused by peer") {
+				fmt.Printf("Could not connect to '%s': connection refused. "+
+					"Check that the service is running and listening on port %d inside the container.\n",
+					remoteAddr, opts.remotePort)
+				return
 			}
 
-	// Run the proxy in the background and signal when it has fully shut down.
-	done := make(chan struct{})
-	go func() {
-		p.Run(ctx)
-		close(done)
-	}()
+			fmt.Printf("Failed to proxy a connection to '%s': %v\n", remoteAddr, err)
+		},
+	}
 
 	// Prefix the local address with the scheme for common HTTP ports so it becomes control-clickable in most
 	// terminals. We assume plain HTTP since TLS is typically terminated by Caddy in front of the service.
 	fmt.Printf("%s%s → %s (%s%s%s)\n", schemeForPort(opts.remotePort), p.Listener.Addr().String(),
 		remoteAddr, opts.service, tui.Faint.Render("/"), containerID)
 
-	<-ctx.Done()
-	// Wait for the proxy to drain in-flight connections and shut down gracefully.
-	<-done
-
+	if err = p.Run(ctx); err != nil {
+		return fmt.Errorf("run proxy to '%s': %w", remoteAddr, err)
+	}
 	return nil
 }
 

cmd/uc/ps.go

@@ -187,7 +187,7 @@ func printContainers(containers []containerInfo) error {
 }
 
 func collectContainers(ctx context.Context, cli *client.Client) ([]containerInfo, error) {
-	listCtx := cli.ProxyMachinesContext(ctx, nil)
+	listCtx := client.ProxyMachinesContext(ctx, nil)
 
 	// List all service containers across all machines in the cluster.
 	machineContainers, err := cli.Docker.ListServiceContainers(

internal/cli/completion/context.go

@@ -11,6 +11,11 @@ import (
 )
 
 func Contexts(ctx context.Context, uncli *cli.CLI, args []string, toComplete string) ([]cobra.Completion, cobra.ShellCompDirective) {
+	// There are no contexts to complete when the CLI uses a direct machine connection (--connect) without a config.
+	if uncli.Config == nil {
+		return nil, cobra.ShellCompDirectiveNoFileComp
+	}
+
 	contexts := slices.Sorted(maps.Keys(uncli.Config.Contexts))
 
 	names := []cobra.Completion{}
@@ -21,7 +26,6 @@ func Contexts(ctx context.Context, uncli *cli.CLI, args []string, toComplete str
 		if strings.HasPrefix(context, toComplete) {
 			names = append(names, context)
 		}
-		names = append(names, context)
 	}
 
 	return names, cobra.ShellCompDirectiveNoFileComp

internal/cli/completion/machine.go

@@ -10,7 +10,8 @@ import (
 )
 
 func Machines(ctx context.Context, uncli *cli.CLI, args []string, toComplete string) ([]cobra.Completion, cobra.ShellCompDirective) {
-	client, err := uncli.ConnectCluster(ctx)
+	// Disable the connection progress output to not interfere with the shell completion output.
+	client, err := uncli.ConnectClusterWithOptions(ctx, cli.ConnectOptions{})
 	if err != nil {
 		return nil, cobra.ShellCompDirectiveError
 	}

internal/cli/completion/service.go

@@ -11,7 +11,8 @@ import (
 )
 
 func Services(ctx context.Context, uncli *cli.CLI, args []string, toComplete string) ([]cobra.Completion, cobra.ShellCompDirective) {
-	client, err := uncli.ConnectCluster(ctx)
+	// Disable the connection progress output to not interfere with the shell completion output.
+	client, err := uncli.ConnectClusterWithOptions(ctx, cli.ConnectOptions{})
 	if err != nil {
 		return nil, cobra.ShellCompDirectiveError
 	}

internal/cli/completion/volume.go

@@ -11,7 +11,8 @@ import (
 )
 
 func Volumes(ctx context.Context, uncli *cli.CLI, args []string, toComplete string) ([]cobra.Completion, cobra.ShellCompDirective) {
-	client, err := uncli.ConnectCluster(ctx)
+	// Disable the connection progress output to not interfere with the shell completion output.
+	client, err := uncli.ConnectClusterWithOptions(ctx, cli.ConnectOptions{})
 	if err != nil {
 		return nil, cobra.ShellCompDirectiveError
 	}

internal/machine/docker/server.go

@@ -563,6 +563,8 @@ func (s *Server) CreateServiceContainer(
 			api.LabelManaged:     "",
 		},
 		User:      spec.Container.User,
+		Tty:       spec.Container.Tty,
+		OpenStdin: spec.Container.OpenStdin,
 	}
 	if spec.Mode == "" {
 		config.Labels[api.LabelServiceMode] = api.ServiceModeReplicated

internal/machine/docker/service.go

@@ -1,12 +1,14 @@
 package docker
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"strconv"
 	"strings"
@@ -233,11 +235,17 @@ func (s *Service) ListImages(ctx context.Context, opts image.ListOptions) (Image
 	return imagesResp, nil
 }
 
-// ContainerLogs streams logs from a container and returns demultiplexed entries via a channel.
+// ContainerLogs streams logs from a container and returns entries via a channel.
 // The channel is closed when streaming completes or context is cancelled.
 func (s *Service) ContainerLogs(
 	ctx context.Context, containerID string, opts api.ServiceLogsOptions,
 ) (<-chan api.LogEntry, error) {
+	ctr, err := s.Client.ContainerInspect(ctx, containerID)
+	if err != nil {
+		return nil, fmt.Errorf("inspect container '%s': %w", containerID, err)
+	}
+	isTTY := ctr.Config != nil && ctr.Config.Tty
+
 	dockerOpts := container.LogsOptions{
 		ShowStdout: true,
 		ShowStderr: true,
@@ -257,25 +265,31 @@ func (s *Service) ContainerLogs(
 	stdoutWriter := &logsChannelWriter{ctx: ctx, ch: outCh, isStderr: false}
 	stderrWriter := &logsChannelWriter{ctx: ctx, ch: outCh, isStderr: true}
 
-	// Wrap the context in a cancellable one to unblock the second goroutine below when StdCopy completes.
+	// Wrap the context in a cancellable one to unblock the second goroutine when log copying completes.
 	ctx, cancel := context.WithCancel(ctx)
 
-	// Run StdCopy in a goroutine to be able to handle context cancellation.
+	// Copy logs in a goroutine to be able to handle context cancellation.
 	go func() {
 		defer close(outCh)
 		defer cancel()
 
-		// StdCopy is blocking and will return when the reader is closed in another goroutine below or on error.
-		if _, err := stdcopy.StdCopy(stdoutWriter, stderrWriter, reader); err != nil {
+		// Docker returns raw stdout for TTY containers and multiplexed stdout/stderr otherwise.
+		var err error
+		if isTTY {
+			_, err = copyRawContainerLogs(stdoutWriter, reader)
+		} else {
+			_, err = stdcopy.StdCopy(stdoutWriter, stderrWriter, reader)
+		}
+		if err != nil {
 			// Send error as the last entry.
 			select {
-			case outCh <- api.LogEntry{Err: fmt.Errorf("demultiplex container logs: %w", err)}:
+			case outCh <- api.LogEntry{Err: fmt.Errorf("copy container logs: %w", err)}:
 			case <-ctx.Done():
 			}
 		}
 	}()
 
-	// Close the reader when the context is done to cancel StdCopy if it's still running.
+	// Close the reader when the context is done to cancel log copying if it's still running.
 	go func() {
 		<-ctx.Done()
 		reader.Close()
@@ -284,7 +298,32 @@ func (s *Service) ContainerLogs(
 	return outCh, nil
 }
 
-// logsChannelWriter is a writer for stdcopy.StdCopy that sends demultiplexed container logs to a channel.
+// copyRawContainerLogs copies a raw TTY log stream one line at a time so each write produces one log entry.
+func copyRawContainerLogs(dst io.Writer, src io.Reader) (written int64, _ error) {
+	reader := bufio.NewReader(src)
+	for {
+		line, readErr := reader.ReadBytes('\n')
+		if len(line) > 0 {
+			n, writeErr := dst.Write(line)
+			written += int64(n)
+			if writeErr != nil {
+				return written, writeErr
+			}
+			if n != len(line) {
+				return written, io.ErrShortWrite
+			}
+		}
+
+		if readErr != nil {
+			if errors.Is(readErr, io.EOF) {
+				return written, nil
+			}
+			return written, readErr
+		}
+	}
+}
+
+// logsChannelWriter sends container log writes to a channel.
 type logsChannelWriter struct {
 	ctx      context.Context
 	ch       chan<- api.LogEntry

internal/machine/docker/service_test.go

@@ -0,0 +1,122 @@
+package docker
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+	dockerclient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestServiceContainerLogs(t *testing.T) {
+	t.Parallel()
+
+	const (
+		containerID = "container-id"
+		firstLog    = "2025-01-01T00:00:00.000000000Z first message\n"
+		secondLog   = "2025-01-01T00:00:01.000000000Z second message\n"
+	)
+
+	var multiplexedLogs bytes.Buffer
+	_, err := stdcopy.NewStdWriter(&multiplexedLogs, stdcopy.Stdout).Write([]byte(firstLog))
+	require.NoError(t, err)
+	_, err = stdcopy.NewStdWriter(&multiplexedLogs, stdcopy.Stderr).Write([]byte(secondLog))
+	require.NoError(t, err)
+
+	tests := []struct {
+		name     string
+		tty      bool
+		logs     []byte
+		streams  []api.LogStreamType
+		messages []string
+	}{
+		{
+			name:     "TTY raw stream",
+			tty:      true,
+			logs:     []byte(firstLog + secondLog),
+			streams:  []api.LogStreamType{api.LogStreamStdout, api.LogStreamStdout},
+			messages: []string{"first message\n", "second message\n"},
+		},
+		{
+			name:     "non-TTY multiplexed stream",
+			logs:     multiplexedLogs.Bytes(),
+			streams:  []api.LogStreamType{api.LogStreamStdout, api.LogStreamStderr},
+			messages: []string{"first message\n", "second message\n"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			dockerClient := newLogsTestClient(t, tt.tty, tt.logs)
+			service := NewService(dockerClient, nil)
+
+			logsCh, err := service.ContainerLogs(context.Background(), containerID, api.ServiceLogsOptions{})
+			require.NoError(t, err)
+
+			var entries []api.LogEntry
+			for entry := range logsCh {
+				require.NoError(t, entry.Err)
+				entries = append(entries, entry)
+			}
+
+			require.Len(t, entries, len(tt.messages))
+			for i := range entries {
+				assert.Equal(t, tt.streams[i], entries[i].Stream)
+				assert.Equal(t, tt.messages[i], string(entries[i].Message))
+				assert.False(t, entries[i].Timestamp.IsZero())
+			}
+		})
+	}
+}
+
+func newLogsTestClient(t *testing.T, tty bool, logs []byte) *dockerclient.Client {
+	t.Helper()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/containers/container-id/json"):
+			w.Header().Set("Content-Type", "application/json")
+			if err := json.NewEncoder(w).Encode(container.InspectResponse{
+				ContainerJSONBase: &container.ContainerJSONBase{ID: "container-id"},
+				Config:            &container.Config{Tty: tty},
+			}); err != nil {
+				t.Errorf("encode inspect response: %v", err)
+			}
+		case strings.HasSuffix(r.URL.Path, "/containers/container-id/logs"):
+			w.Header().Set("Content-Type", "application/vnd.docker.raw-stream")
+			if _, err := w.Write(logs); err != nil {
+				t.Errorf("write logs response: %v", err)
+			}
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(server.Close)
+
+	serverURL, err := url.Parse(server.URL)
+	require.NoError(t, err)
+
+	dockerClient, err := dockerclient.NewClientWithOpts(
+		dockerclient.WithHost("tcp://"+serverURL.Host),
+		dockerclient.WithHTTPClient(server.Client()),
+		dockerclient.WithVersion("1.48"),
+	)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, dockerClient.Close())
+	})
+
+	return dockerClient
+}

internal/machine/machine.go

@@ -42,6 +42,8 @@ import (
 	"github.com/psviderski/uncloud/internal/secret"
 	"github.com/psviderski/uncloud/internal/version"
 	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/psviderski/uncloud/pkg/distlock"
+	distlockgrpc "github.com/psviderski/uncloud/pkg/distlock/grpc"
 	"github.com/psviderski/unregistry"
 	"github.com/siderolabs/grpc-proxy/proxy"
 	"golang.org/x/sync/errgroup"
@@ -305,7 +307,8 @@ func NewMachine(config *Config) (*Machine, error) {
 		WaitForNetworkReady: m.WaitForNetworkReady,
 	})
 	caddyServer := caddyconfig.NewServer(caddyconfig.NewService(config.CaddyConfigDir))
-	m.localMachineServer = newGRPCServer(m, c, m.dockerServer, caddyServer)
+	leaseServer := distlockgrpc.NewServer(distlock.NewMemoryStore())
+	m.localMachineServer = newGRPCServer(m, c, m.dockerServer, caddyServer, leaseServer)
 
 	if m.Initialised() {
 		close(m.initialised)
@@ -314,12 +317,19 @@ func NewMachine(config *Config) (*Machine, error) {
 	return m, nil
 }
 
-func newGRPCServer(m pb.MachineServer, c pb.ClusterServer, d pb.DockerServer, caddy pb.CaddyServer) *grpc.Server {
+func newGRPCServer(
+	m pb.MachineServer,
+	c pb.ClusterServer,
+	d pb.DockerServer,
+	caddy pb.CaddyServer,
+	lease distlockgrpc.LeaseServer,
+) *grpc.Server {
 	s := grpc.NewServer()
 	pb.RegisterMachineServer(s, m)
 	pb.RegisterClusterServer(s, c)
 	pb.RegisterDockerServer(s, d)
 	pb.RegisterCaddyServer(s, caddy)
+	distlockgrpc.RegisterLeaseServer(s, lease)
 	return s
 }
 

internal/proxy/proxy.go

@@ -2,11 +2,12 @@ package proxy
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net"
-	"os"
 	"sync"
+	"syscall"
 	"time"
 )
 
@@ -15,58 +16,53 @@ type Proxy struct {
 	Listener    net.Listener
 	RemoteAddr  string
 	DialContext func(ctx context.Context, network, address string) (net.Conn, error)
+	// OnError is called for errors that occur during proxying individual connections. It may be called concurrently
+	// for different connections.
 	OnError     func(error)
 	activeConns sync.WaitGroup
 }
 
-// deadliner is an interface for listeners that support setting deadlines.
-type deadliner interface {
-	SetDeadline(t time.Time) error
-}
-
 // halfCloser is an interface for connections that support half-close.
 type halfCloser interface {
 	CloseWrite() error
 }
 
-// Run starts the proxy and runs until the context is canceled.
-func (p *Proxy) Run(ctx context.Context) {
+// IsConnectionClosedError reports whether err indicates that a connection was closed or aborted by either peer.
+// Callers can use it to ignore routine connection shutdown or broken pipe errors reported to Proxy.OnError.
+func IsConnectionClosedError(err error) bool {
+	return errors.Is(err, net.ErrClosed) || errors.Is(err, io.ErrClosedPipe) ||
+		errors.Is(err, syscall.EPIPE) || errors.Is(err, syscall.ECONNRESET)
+}
+
+// Run starts the proxy and runs until the context is canceled or the listener fails. It returns nil when the context
+// is canceled. Errors handling individual connections are reported to OnError and do not stop the proxy.
+func (p *Proxy) Run(ctx context.Context) error {
 	if p.DialContext == nil {
 		p.DialContext = (&net.Dialer{}).DialContext
 	}
 
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	defer p.Listener.Close()
 
-	// Handle incoming connections until context is canceled.
-Loop:
+	// Closing the listener unblocks Accept when the context is canceled. This works for both TCP and Unix listeners
+	// and avoids polling with listener deadlines.
+	stopClose := context.AfterFunc(ctx, func() {
+		p.Listener.Close()
+	})
+	defer stopClose()
+
+	var runErr error
 	for {
-		select {
-		case <-ctx.Done():
-			break Loop
-		default:
-		}
-
-		// Set a deadline on the listener if supported to check context periodically.
-		if dl, ok := p.Listener.(deadliner); ok {
-			dl.SetDeadline(time.Now().Add(1 * time.Second))
-		}
-
 		conn, err := p.Listener.Accept()
 		if err != nil {
-			if os.IsTimeout(err) {
-				// Just a timeout, continue to check context and accept again.
-				continue
+			if ctx.Err() != nil {
+				break
 			}
 
-			select {
-			case <-ctx.Done():
-				break Loop
-			default:
-				if p.OnError != nil {
-					p.OnError(fmt.Errorf("accept local connection: %w", err))
-				}
-				continue
-			}
+			runErr = fmt.Errorf("accept local connection: %w", err)
+			cancel()
+			break
 		}
 
 		p.activeConns.Add(1)
@@ -75,6 +71,7 @@ Loop:
 
 	// Wait for all connections to finish.
 	p.activeConns.Wait()
+	return runErr
 }
 
 func (p *Proxy) handleConnection(ctx context.Context, localConn net.Conn) {
@@ -87,46 +84,62 @@ func (p *Proxy) handleConnection(ctx context.Context, localConn net.Conn) {
 
 	remoteConn, err := p.DialContext(dialCtx, "tcp", p.RemoteAddr)
 	if err != nil {
-		if p.OnError != nil {
+		if ctx.Err() == nil && p.OnError != nil {
 			p.OnError(fmt.Errorf("connect remote address '%s': %w", p.RemoteAddr, err))
 		}
 		return
 	}
 	defer remoteConn.Close()
 
-	// Bidirectional copy with proper half-close handling.
+	// Closing both connections aborts both copies after cancellation or a copy error. A clean EOF still uses
+	// half-close so the other direction can finish sending any remaining data.
+	closeConnections := func() {
+		localConn.Close()
+		remoteConn.Close()
+	}
+	stopClose := context.AfterFunc(ctx, closeConnections)
+	defer stopClose()
+
 	done := make(chan error, 2)
 
 	go func() {
 		_, err := io.Copy(remoteConn, localConn)
+		if err != nil {
+			done <- err
+			closeConnections()
+			return
+		}
 		// Close write half of remote connection if supported.
 		if hc, ok := remoteConn.(halfCloser); ok {
 			hc.CloseWrite()
 		}
-		done <- err
+		done <- nil
 	}()
 
 	go func() {
 		_, err := io.Copy(localConn, remoteConn)
+		if err != nil {
+			done <- err
+			closeConnections()
+			return
+		}
 		// Close write half of local connection if supported.
 		if hc, ok := localConn.(halfCloser); ok {
 			hc.CloseWrite()
 		}
-		done <- err
+		done <- nil
 	}()
 
-	// Wait for both copies to complete or context cancel.
+	// Wait for both copies to complete. The first error is the original failure because a copy reports it before
+	// closing the connections to unblock the other copy.
+	var copyErr error
 	for range 2 {
-		select {
-		case <-ctx.Done():
-			// Close connections to abort ongoing copies.
-			localConn.Close()
-			remoteConn.Close()
-			return
-		case err = <-done:
-			if err != nil && p.OnError != nil {
-				p.OnError(fmt.Errorf("data copy: %w", err))
+		if err = <-done; err != nil && copyErr == nil {
+			copyErr = err
 		}
 	}
+
+	if copyErr != nil && ctx.Err() == nil && p.OnError != nil {
+		p.OnError(fmt.Errorf("data copy: %w", copyErr))
 	}
 }

internal/proxy/proxy_test.go

@@ -0,0 +1,188 @@
+package proxy
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsConnectionClosedError(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{name: "closed network connection", err: net.ErrClosed, want: true},
+		{name: "closed pipe", err: io.ErrClosedPipe, want: true},
+		{name: "broken pipe", err: syscall.EPIPE, want: true},
+		{name: "connection reset", err: syscall.ECONNRESET, want: true},
+		{name: "wrapped connection error", err: fmt.Errorf("copy data: %w", syscall.EPIPE), want: true},
+		{name: "other error", err: errors.New("copy failed"), want: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.want, IsConnectionClosedError(tt.err))
+		})
+	}
+}
+
+func TestRunContinuesAfterClosedConnectionError(t *testing.T) {
+	t.Parallel()
+
+	listener := newTestListener()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	closedErrCh := make(chan error, 1)
+	unexpectedErrCh := make(chan error, 1)
+	var dialCount atomic.Int32
+
+	p := &Proxy{
+		Listener:   listener,
+		RemoteAddr: "remote:80",
+		DialContext: func(context.Context, string, string) (net.Conn, error) {
+			if dialCount.Add(1) == 1 {
+				return readErrorConn{err: syscall.EPIPE}, nil
+			}
+
+			proxyConn, upstreamConn := net.Pipe()
+			go func() {
+				defer upstreamConn.Close()
+				_, _ = upstreamConn.Write([]byte("ok"))
+			}()
+			return proxyConn, nil
+		},
+		OnError: func(err error) {
+			if IsConnectionClosedError(err) {
+				closedErrCh <- err
+				return
+			}
+			unexpectedErrCh <- err
+		},
+	}
+
+	runErrCh := make(chan error, 1)
+	go func() {
+		runErrCh <- p.Run(ctx)
+	}()
+
+	firstConn := listener.connect()
+	defer firstConn.Close()
+
+	select {
+	case connErr := <-closedErrCh:
+		require.ErrorIs(t, connErr, syscall.EPIPE)
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for closed connection error")
+	}
+
+	secondConn := listener.connect()
+	defer secondConn.Close()
+	require.NoError(t, secondConn.SetReadDeadline(time.Now().Add(time.Second)))
+
+	got := make([]byte, 2)
+	_, err := io.ReadFull(secondConn, got)
+	require.NoError(t, err)
+	require.Equal(t, "ok", string(got))
+
+	select {
+	case unexpectedErr := <-unexpectedErrCh:
+		t.Fatalf("unexpected connection error: %v", unexpectedErr)
+	default:
+	}
+
+	cancel()
+	select {
+	case runErr := <-runErrCh:
+		require.NoError(t, runErr)
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for proxy to stop")
+	}
+}
+
+func TestRunReturnsListenerError(t *testing.T) {
+	t.Parallel()
+
+	listenerErr := errors.New("listener failed")
+	listener := errorListener{err: listenerErr}
+
+	p := &Proxy{Listener: listener}
+	err := p.Run(context.Background())
+	require.Error(t, err)
+	require.ErrorContains(t, err, "accept local connection")
+	require.ErrorIs(t, err, listenerErr)
+}
+
+type testListener struct {
+	conns     chan net.Conn
+	closed    chan struct{}
+	closeOnce sync.Once
+}
+
+func newTestListener() *testListener {
+	return &testListener{
+		conns:  make(chan net.Conn),
+		closed: make(chan struct{}),
+	}
+}
+
+func (l *testListener) connect() net.Conn {
+	clientConn, proxyConn := net.Pipe()
+	l.conns <- proxyConn
+	return clientConn
+}
+
+func (l *testListener) Accept() (net.Conn, error) {
+	select {
+	case conn := <-l.conns:
+		return conn, nil
+	case <-l.closed:
+		return nil, net.ErrClosed
+	}
+}
+
+func (l *testListener) Close() error {
+	l.closeOnce.Do(func() {
+		close(l.closed)
+	})
+	return nil
+}
+
+func (l *testListener) Addr() net.Addr {
+	return &net.TCPAddr{}
+}
+
+type errorListener struct {
+	err error
+}
+
+func (l errorListener) Accept() (net.Conn, error) { return nil, l.err }
+func (errorListener) Close() error                { return nil }
+func (errorListener) Addr() net.Addr              { return &net.TCPAddr{} }
+
+// readErrorConn fails reads immediately so tests can deterministically exercise a proxy copy failure.
+type readErrorConn struct {
+	err error
+}
+
+func (c readErrorConn) Read([]byte) (int, error)       { return 0, c.err }
+func (readErrorConn) Write(p []byte) (int, error)      { return len(p), nil }
+func (readErrorConn) Close() error                     { return nil }
+func (readErrorConn) LocalAddr() net.Addr              { return &net.TCPAddr{} }
+func (readErrorConn) RemoteAddr() net.Addr             { return &net.TCPAddr{} }
+func (readErrorConn) SetDeadline(time.Time) error      { return nil }
+func (readErrorConn) SetReadDeadline(time.Time) error  { return nil }
+func (readErrorConn) SetWriteDeadline(time.Time) error { return nil }

mise.toml

@@ -44,7 +44,7 @@ run = "mise lock --platform linux-x64,macos-arm64"
 description = "Regenerate gRPC API code from .proto files"
 run = """
 protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative \
-  --proto_path=. --proto_path=internal/machine/api/vendor internal/machine/api/pb/*.proto
+  --proto_path=. --proto_path=internal/machine/api/vendor internal/machine/api/pb/*.proto pkg/distlock/grpc/*.proto
 """
 
 [tasks.uc]

pkg/api/service.go

@@ -265,6 +265,11 @@ type ContainerSpec struct {
 	LogDriver *LogDriver
 	// PidMode sets the PID namespace mode for the container. Currently only "" or "host" is supported.
 	PidMode string
+	// Tty allocates a pseudo-TTY and connects the container's standard streams to it.
+	// Standard output and standard error share one stream.
+	Tty bool
+	// OpenStdin allocates standard input and keeps it open.
+	OpenStdin bool
 	// Privileged gives extended privileges to the container. This is a security risk and should be used with caution.
 	Privileged bool
 	// PullPolicy determines when to pull the image from the registry or use the image already available in the cluster.

pkg/client/client.go

@@ -10,6 +10,7 @@ import (
 	"github.com/psviderski/uncloud/internal/machine/api/pb"
 	"github.com/psviderski/uncloud/internal/machine/docker"
 	"github.com/psviderski/uncloud/pkg/api"
+	distlockgrpc "github.com/psviderski/uncloud/pkg/distlock/grpc"
 	"golang.org/x/net/proxy"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
@@ -28,6 +29,7 @@ type Client struct {
 	// Docker is a namespaced client for the Docker service to distinguish Uncloud-specific service container operations
 	// from generic Docker operations.
 	Docker *docker.Client
+	leases distlockgrpc.LeaseClient
 }
 
 var _ api.Client = (*Client)(nil)
@@ -57,6 +59,7 @@ func New(ctx context.Context, connector Connector) (*Client, error) {
 	c.ClusterClient = pb.NewClusterClient(c.conn)
 	c.Caddy = pb.NewCaddyClient(c.conn)
 	c.Docker = docker.NewClient(c.conn)
+	c.leases = distlockgrpc.NewLeaseClient(c.conn)
 
 	return c, nil
 }
@@ -78,8 +81,8 @@ func (cli *Client) progressOut() *streams.Out {
 // ProxyMachinesContext returns a new context that proxies gRPC requests to the specified machines.
 // If namesOrIDs is nil or empty, all machines are included.
 // This triggers One2Many proxying, which always injects metadata into the response.
-func (cli *Client) ProxyMachinesContext(ctx context.Context, namesOrIDs []string) context.Context {
-	md := metadata.New(nil)
+func ProxyMachinesContext(ctx context.Context, namesOrIDs []string) context.Context {
+	md := outgoingMetadataWithoutProxyTargets(ctx)
 	if len(namesOrIDs) == 0 {
 		md.Append("machines", "*")
 	} else {
@@ -92,7 +95,26 @@ func (cli *Client) ProxyMachinesContext(ctx context.Context, namesOrIDs []string
 // ProxySingleMachineContext returns a new context that proxies gRPC requests to a single specified machine.
 // This triggers One2One proxying, which does NOT inject metadata into the response.
 // Use this for requests that expect a single response message without metadata wrapper.
-func (cli *Client) ProxySingleMachineContext(ctx context.Context, nameOrID string) context.Context {
-	md := metadata.Pairs("machine", nameOrID)
+func ProxySingleMachineContext(ctx context.Context, nameOrID string) context.Context {
+	md := outgoingMetadataWithoutProxyTargets(ctx)
+	md.Set("machine", nameOrID)
 	return metadata.NewOutgoingContext(ctx, md)
 }
+
+func outgoingMetadataWithoutProxyTargets(ctx context.Context) metadata.MD {
+	md, _ := metadata.FromOutgoingContext(ctx)
+	md = md.Copy()
+	md.Delete("machine")
+	md.Delete("machines")
+	return md
+}
+
+// ProxyMachinesContext returns a new context that proxies gRPC requests to the specified machines.
+func (cli *Client) ProxyMachinesContext(ctx context.Context, namesOrIDs []string) context.Context {
+	return ProxyMachinesContext(ctx, namesOrIDs)
+}
+
+// ProxySingleMachineContext returns a new context that proxies gRPC requests to a single specified machine.
+func (cli *Client) ProxySingleMachineContext(ctx context.Context, nameOrID string) context.Context {
+	return ProxySingleMachineContext(ctx, nameOrID)
+}

pkg/client/client_test.go

@@ -0,0 +1,79 @@
+package client
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/metadata"
+)
+
+func TestProxySingleMachineContext(t *testing.T) {
+	original := metadata.Pairs(
+		"authorization", "token",
+		"machine", "old-machine",
+		"machines", "old-machine-a",
+		"machines", "old-machine-b",
+	)
+	ctx := metadata.NewOutgoingContext(context.Background(), original)
+
+	proxyCtx := ProxySingleMachineContext(ctx, "new-machine")
+
+	md, ok := metadata.FromOutgoingContext(proxyCtx)
+	require.True(t, ok)
+	require.Equal(t, metadata.Pairs(
+		"authorization", "token",
+		"machine", "new-machine",
+	), md)
+	require.Equal(t, metadata.Pairs(
+		"authorization", "token",
+		"machine", "old-machine",
+		"machines", "old-machine-a",
+		"machines", "old-machine-b",
+	), original)
+}
+
+func TestProxyMachinesContext(t *testing.T) {
+	tests := []struct {
+		name     string
+		machines []string
+		want     []string
+	}{
+		{
+			name:     "specified machines",
+			machines: []string{"machine-a", "machine-b"},
+			want:     []string{"machine-a", "machine-b"},
+		},
+		{
+			name: "all machines",
+			want: []string{"*"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			original := metadata.Pairs(
+				"authorization", "token",
+				"machine", "old-machine",
+				"machines", "old-machine-a",
+				"machines", "old-machine-b",
+			)
+			ctx := metadata.NewOutgoingContext(context.Background(), original)
+
+			proxyCtx := ProxyMachinesContext(ctx, tt.machines)
+
+			md, ok := metadata.FromOutgoingContext(proxyCtx)
+			require.True(t, ok)
+			require.Equal(t, metadata.MD{
+				"authorization": {"token"},
+				"machines":      tt.want,
+			}, md)
+			require.Equal(t, metadata.Pairs(
+				"authorization", "token",
+				"machine", "old-machine",
+				"machines", "old-machine-a",
+				"machines", "old-machine-b",
+			), original)
+		})
+	}
+}

pkg/client/compose/service.go

@@ -54,6 +54,8 @@ func ServiceSpecFromCompose(project *types.Project, serviceName string) (api.Ser
 			Image:       service.Image,
 			Init:        service.Init,
 			PidMode:     service.Pid,
+			Tty:         service.Tty,
+			OpenStdin:   service.StdinOpen,
 			Privileged:  service.Privileged,
 			PullPolicy:  pullPolicy,
 			Resources:   resourcesFromCompose(service),

pkg/client/compose/service_test.go

@@ -128,6 +128,8 @@ func TestServiceSpecFromCompose(t *testing.T) {
 							},
 						},
 						PidMode:    "host",
+						Tty:        true,
+						OpenStdin:  true,
 						Privileged: true,
 						PullPolicy: api.PullPolicyAlways,
 						Resources: api.ContainerResources{

pkg/client/compose/testdata/compose-full-spec.yaml

@@ -7,6 +7,8 @@ services:
     command: ["nginx", "updated", "command"]
     cpus: 0.5
     pid: host
+    tty: true
+    stdin_open: true
     deploy:
       update_config:
         order: stop-first

pkg/client/container.go

@@ -74,7 +74,7 @@ func (cli *Client) createServiceContainerWithPull(
 	resp.Name = containerName
 
 	// Proxy Docker gRPC requests to the selected machine.
-	ctx = cli.ProxySingleMachineContext(ctx, machine.Machine.Id)
+	ctx = ProxySingleMachineContext(ctx, machine.Machine.Id)
 
 	pw := progress.ContextWriter(ctx)
 	eventID := cliprogress.NewContainerEventID(ctx, containerName, machine.Machine.Name)
@@ -277,7 +277,7 @@ func (cli *Client) resolveContainerOperation(
 
 	eventID := cliprogress.ContainerEventID(ctx, ctr.Container.ServiceSpec.Name, ctr.Container.ID, ctr.MachineName)
 	return containerOperationContext{
-		ctx:         cli.ProxySingleMachineContext(ctx, ctr.MachineID),
+		ctx:         ProxySingleMachineContext(ctx, ctr.MachineID),
 		containerID: ctr.Container.ID,
 		eventID:     eventID,
 	}, nil
@@ -375,7 +375,7 @@ func (cli *Client) ExecContainer(
 	}
 
 	// Proxy Docker gRPC requests to the machine hosting the container
-	ctx = cli.ProxySingleMachineContext(ctx, machine.Machine.Id)
+	ctx = ProxySingleMachineContext(ctx, machine.Machine.Id)
 
 	// Execute the command in the container
 	exitCode, err := cli.Docker.ExecContainer(ctx, machinedocker.ExecConfig{
@@ -452,7 +452,7 @@ func (cli *Client) WaitContainerHealthy(
 	}
 
 	// For containers with a health check, wait until Docker reports healthy or unhealthy.
-	mctx := cli.ProxySingleMachineContext(ctx, machine.Machine.Id)
+	mctx := ProxySingleMachineContext(ctx, machine.Machine.Id)
 	mctx, cancel := context.WithTimeout(mctx, healthcheckTimeout(mc.Container.Config.Healthcheck))
 	defer cancel()
 	ticker := time.NewTicker(1 * time.Second)

pkg/client/deploy/container_test.go

@@ -66,6 +66,26 @@ func TestEvalContainerSpecChange_ContainerPidMode(t *testing.T) {
 	assert.Equal(t, ContainerNeedsRecreate, EvalContainerSpecChange(newSpec, currentSpec))
 }
 
+func TestEvalContainerSpecChange_ContainerTty(t *testing.T) {
+	t.Parallel()
+
+	currentSpec := api.ServiceSpec{
+		Container: api.ContainerSpec{
+			Image: "nginx:latest",
+		},
+	}
+	newSpec := api.ServiceSpec{
+		Container: api.ContainerSpec{
+			Image:     "nginx:latest",
+			Tty:       true,
+			OpenStdin: true,
+		},
+	}
+
+	assert.Equal(t, ContainerNeedsRecreate, EvalContainerSpecChange(currentSpec, newSpec))
+	assert.Equal(t, ContainerNeedsRecreate, EvalContainerSpecChange(newSpec, currentSpec))
+}
+
 func TestEvalContainerSpecChange_ContainerResources(t *testing.T) {
 	t.Parallel()
 

pkg/client/image.go

@@ -59,7 +59,7 @@ func (cli *Client) InspectRemoteImage(ctx context.Context, id string) ([]api.Mac
 // it lists images on all machines.
 func (cli *Client) ListImages(ctx context.Context, filter api.ImageFilter) ([]api.MachineImages, error) {
 	// Broadcast the image list request to the specified machines or all machines if none specified.
-	listCtx := cli.ProxyMachinesContext(ctx, filter.Machines)
+	listCtx := ProxyMachinesContext(ctx, filter.Machines)
 
 	opts := image.ListOptions{Manifests: true}
 	if filter.Name != "" {
@@ -252,13 +252,19 @@ func (cli *Client) pushImageToMachine(
 	// The proxy runs in a goroutine. Capture the first error in a channel
 	// so we can surface it alongside the push error if push fails.
 	proxyErrCh := make(chan error, 1)
-	onProxyError := func(err error) {
+	recordProxyError := func(err error) {
 		select {
 		case proxyErrCh <- fmt.Errorf("proxy to unregistry: %w", err):
 		default:
 		}
 		pw.Event(progress.NewEvent(proxyEventID, progress.Error, err.Error()))
 	}
+	onProxyError := func(err error) {
+		if proxy.IsConnectionClosedError(err) {
+			return
+		}
+		recordProxyError(err)
+	}
 
 	// socketPath is set for plain rootless Docker (not running inside a VM): the Go proxy listens on a unix
 	// socket that is bind-mounted into the socat container, bypassing slirp4netns network routing entirely.
@@ -315,7 +321,11 @@ func (cli *Client) pushImageToMachine(
 	}
 	defer cleanup()
 
-	go unregProxy.Run(proxyCtx)
+	go func() {
+		if err := unregProxy.Run(proxyCtx); err != nil {
+			recordProxyError(err)
+		}
+	}()
 
 	if dockerEnv.Virtualised {
 		// VM-based Docker (Docker Desktop, Rancher Desktop, etc.): run a socat container inside the VM

pkg/client/locker.go

@@ -0,0 +1,95 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/psviderski/uncloud/pkg/distlock"
+	distlockgrpc "github.com/psviderski/uncloud/pkg/distlock/grpc"
+	"google.golang.org/protobuf/types/known/durationpb"
+)
+
+// NewLocker creates a distlock.Locker that acquires automatically renewed distributed leases over the machines
+// in the cluster. The Locker uses the client's connection, so callers must release its active leases before closing
+// the client.
+func (cli *Client) NewLocker(config distlock.Config) (*distlock.Locker, error) {
+	return distlock.New(&lockCluster{client: cli}, config)
+}
+
+type lockCluster struct {
+	client *Client
+}
+
+var _ distlock.Cluster = (*lockCluster)(nil)
+
+// Nodes returns a point-in-time snapshot of the registered machines in the cluster, including temporarily unavailable
+// ones. The Locker calls Nodes at the start of each Acquire and retains the returned snapshot across acquisition
+// retries and for the lifetime of any acquired lease.
+//
+// Adding or removing machines, combined with eventual replication of the machine list, can cause different
+// acquisitions to use different snapshots while active leases continue using older ones. This adapter does not
+// version snapshots or coordinate membership transitions. If old and new snapshots allow disjoint quorums, two
+// clients can acquire leases for the same resource. Membership changes must preserve quorum overlap while leases from
+// older snapshots may remain valid.
+func (c *lockCluster) Nodes(ctx context.Context) ([]distlock.Node, error) {
+	machines, err := c.client.ListMachines(ctx, nil)
+	if err != nil {
+		return nil, fmt.Errorf("list machines: %w", err)
+	}
+
+	nodes := make([]distlock.Node, 0, len(machines))
+	for _, m := range machines {
+		nodes = append(nodes, &lockNode{
+			id:     m.Machine.Id,
+			leases: c.client.leases,
+		})
+	}
+	return nodes, nil
+}
+
+type lockNode struct {
+	id     string
+	leases distlockgrpc.LeaseClient
+}
+
+var _ distlock.Node = (*lockNode)(nil)
+
+func (n *lockNode) Acquire(
+	ctx context.Context, resource string, token []byte, ttl time.Duration,
+) (bool, error) {
+	resp, err := n.leases.Acquire(ProxySingleMachineContext(ctx, n.id), &distlockgrpc.AcquireLeaseRequest{
+		Resource: resource,
+		Token:    token,
+		Ttl:      durationpb.New(ttl),
+	})
+	if err != nil {
+		return false, err
+	}
+	return resp.Acquired, nil
+}
+
+func (n *lockNode) Renew(
+	ctx context.Context, resource string, token []byte, ttl time.Duration,
+) (bool, error) {
+	resp, err := n.leases.Renew(ProxySingleMachineContext(ctx, n.id), &distlockgrpc.RenewLeaseRequest{
+		Resource: resource,
+		Token:    token,
+		Ttl:      durationpb.New(ttl),
+	})
+	if err != nil {
+		return false, err
+	}
+	return resp.Renewed, nil
+}
+
+func (n *lockNode) Release(ctx context.Context, resource string, token []byte) (bool, error) {
+	resp, err := n.leases.Release(ProxySingleMachineContext(ctx, n.id), &distlockgrpc.ReleaseLeaseRequest{
+		Resource: resource,
+		Token:    token,
+	})
+	if err != nil {
+		return false, err
+	}
+	return resp.Released, nil
+}

pkg/client/logs.go

@@ -102,7 +102,7 @@ func (cli *Client) ServiceLogs(
 func (cli *Client) ContainerLogs(
 	ctx context.Context, machineNameOrID string, containerID string, opts api.ServiceLogsOptions,
 ) (<-chan api.LogEntry, error) {
-	proxyCtx := cli.ProxySingleMachineContext(ctx, machineNameOrID)
+	proxyCtx := ProxySingleMachineContext(ctx, machineNameOrID)
 
 	req := &pb.LogsRequest{
 		Id:     containerID,
@@ -198,7 +198,7 @@ func (cli *Client) MachineLogs(
 func (cli *Client) systemServiceLogs(
 	ctx context.Context, machineID, service string, opts api.ServiceLogsOptions,
 ) (<-chan api.LogEntry, error) {
-	proxyCtx := cli.ProxySingleMachineContext(ctx, machineID)
+	proxyCtx := ProxySingleMachineContext(ctx, machineID)
 
 	req := &pb.LogsRequest{
 		Id:     service,

pkg/client/machine.go

@@ -78,7 +78,7 @@ func (cli *Client) ListMachines(ctx context.Context, filter *api.MachineFilter)
 func (cli *Client) UpdateMachine(
 	ctx context.Context, nameOrID string, req *pb.UpdateMachineRequest,
 ) (*pb.MachineInfo, error) {
-	ctx = cli.ProxySingleMachineContext(ctx, nameOrID)
+	ctx = ProxySingleMachineContext(ctx, nameOrID)
 	resp, err := cli.MachineClient.UpdateMachine(ctx, req)
 	if err != nil {
 		if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound {

pkg/client/volume.go

@@ -27,7 +27,7 @@ func (cli *Client) CreateVolume(
 		return resp, fmt.Errorf("inspect machine '%s': %w", machineNameOrID, err)
 	}
 	// Proxy Docker gRPC requests to the selected machine.
-	ctx = cli.ProxySingleMachineContext(ctx, machine.Machine.Id)
+	ctx = ProxySingleMachineContext(ctx, machine.Machine.Id)
 
 	pw := progress.ContextWriter(ctx)
 	eventID := cliprogress.VolumeEventID(opts.Name, machine.Machine.Name)
@@ -56,7 +56,7 @@ func (cli *Client) ListVolumes(ctx context.Context, filter *api.VolumeFilter) ([
 		proxyMachines = filter.Machines
 	}
 
-	listCtx := cli.ProxyMachinesContext(ctx, proxyMachines)
+	listCtx := ProxyMachinesContext(ctx, proxyMachines)
 	machineVolumes, err := cli.Docker.ListVolumes(listCtx, volume.ListOptions{})
 	if err != nil {
 		return nil, err
@@ -107,7 +107,7 @@ func (cli *Client) RemoveVolume(ctx context.Context, machineNameOrID, volumeName
 		return fmt.Errorf("inspect machine '%s': %w", machineNameOrID, err)
 	}
 	// Proxy Docker gRPC requests to the selected machine.
-	ctx = cli.ProxySingleMachineContext(ctx, machine.Machine.Id)
+	ctx = ProxySingleMachineContext(ctx, machine.Machine.Id)
 
 	pw := progress.ContextWriter(ctx)
 	eventID := cliprogress.VolumeEventID(volumeName, machine.Machine.Name)

pkg/distlock/doc.go

@@ -0,0 +1,12 @@
+// Package distlock provides distributed, automatically renewed leases across independent nodes.
+//
+// Its quorum and lease semantics are based on the Redlock algorithm described at
+// https://redis.io/docs/latest/develop/clients/patterns/distributed-locks/. The core package is independent of storage
+// and network transport and does not require Redis. Applications that communicate with remote nodes over gRPC can use
+// the grpc subpackage to expose and call node-local lease operations.
+//
+// A Cluster must return every node in the lock group, including temporarily unavailable nodes, because every node
+// counts toward quorum. Changing the node set is unsafe if a new quorum can be disjoint from an earlier quorum while
+// leases acquired from the earlier node set may still be valid. Callers must stop protected work when the context
+// returned by Lease.Context is done.
+package distlock

pkg/distlock/grpc/lease.pb.go

@@ -0,0 +1,547 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.34.2
+// 	protoc        v5.27.3
+// source: pkg/distlock/grpc/lease.proto
+
+package grpc
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type AcquireLeaseRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Resource string `protobuf:"bytes,1,opt,name=resource,proto3" json:"resource,omitempty"`
+	// Token uniquely identifies the lease owner.
+	Token []byte `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"`
+	// TTL sets how long the lease remains valid without renewal.
+	Ttl *durationpb.Duration `protobuf:"bytes,3,opt,name=ttl,proto3" json:"ttl,omitempty"`
+}
+
+func (x *AcquireLeaseRequest) Reset() {
+	*x = AcquireLeaseRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *AcquireLeaseRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AcquireLeaseRequest) ProtoMessage() {}
+
+func (x *AcquireLeaseRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use AcquireLeaseRequest.ProtoReflect.Descriptor instead.
+func (*AcquireLeaseRequest) Descriptor() ([]byte, []int) {
+	return file_pkg_distlock_grpc_lease_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *AcquireLeaseRequest) GetResource() string {
+	if x != nil {
+		return x.Resource
+	}
+	return ""
+}
+
+func (x *AcquireLeaseRequest) GetToken() []byte {
+	if x != nil {
+		return x.Token
+	}
+	return nil
+}
+
+func (x *AcquireLeaseRequest) GetTtl() *durationpb.Duration {
+	if x != nil {
+		return x.Ttl
+	}
+	return nil
+}
+
+type AcquireLeaseResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Acquired is true when this request successfully created the lease.
+	Acquired bool `protobuf:"varint,1,opt,name=acquired,proto3" json:"acquired,omitempty"`
+}
+
+func (x *AcquireLeaseResponse) Reset() {
+	*x = AcquireLeaseResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *AcquireLeaseResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AcquireLeaseResponse) ProtoMessage() {}
+
+func (x *AcquireLeaseResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use AcquireLeaseResponse.ProtoReflect.Descriptor instead.
+func (*AcquireLeaseResponse) Descriptor() ([]byte, []int) {
+	return file_pkg_distlock_grpc_lease_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *AcquireLeaseResponse) GetAcquired() bool {
+	if x != nil {
+		return x.Acquired
+	}
+	return false
+}
+
+type RenewLeaseRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Resource string `protobuf:"bytes,1,opt,name=resource,proto3" json:"resource,omitempty"`
+	// Token identifies the owner of the existing lease.
+	Token []byte `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"`
+	// TTL sets how long the renewed lease remains valid.
+	Ttl *durationpb.Duration `protobuf:"bytes,3,opt,name=ttl,proto3" json:"ttl,omitempty"`
+}
+
+func (x *RenewLeaseRequest) Reset() {
+	*x = RenewLeaseRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RenewLeaseRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RenewLeaseRequest) ProtoMessage() {}
+
+func (x *RenewLeaseRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RenewLeaseRequest.ProtoReflect.Descriptor instead.
+func (*RenewLeaseRequest) Descriptor() ([]byte, []int) {
+	return file_pkg_distlock_grpc_lease_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *RenewLeaseRequest) GetResource() string {
+	if x != nil {
+		return x.Resource
+	}
+	return ""
+}
+
+func (x *RenewLeaseRequest) GetToken() []byte {
+	if x != nil {
+		return x.Token
+	}
+	return nil
+}
+
+func (x *RenewLeaseRequest) GetTtl() *durationpb.Duration {
+	if x != nil {
+		return x.Ttl
+	}
+	return nil
+}
+
+type RenewLeaseResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Renewed is true when an unexpired lease matched the ownership token and was successfully renewed.
+	Renewed bool `protobuf:"varint,1,opt,name=renewed,proto3" json:"renewed,omitempty"`
+}
+
+func (x *RenewLeaseResponse) Reset() {
+	*x = RenewLeaseResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RenewLeaseResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RenewLeaseResponse) ProtoMessage() {}
+
+func (x *RenewLeaseResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RenewLeaseResponse.ProtoReflect.Descriptor instead.
+func (*RenewLeaseResponse) Descriptor() ([]byte, []int) {
+	return file_pkg_distlock_grpc_lease_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *RenewLeaseResponse) GetRenewed() bool {
+	if x != nil {
+		return x.Renewed
+	}
+	return false
+}
+
+type ReleaseLeaseRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Resource string `protobuf:"bytes,1,opt,name=resource,proto3" json:"resource,omitempty"`
+	// Token identifies the owner of the existing lease.
+	Token []byte `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"`
+}
+
+func (x *ReleaseLeaseRequest) Reset() {
+	*x = ReleaseLeaseRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ReleaseLeaseRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ReleaseLeaseRequest) ProtoMessage() {}
+
+func (x *ReleaseLeaseRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ReleaseLeaseRequest.ProtoReflect.Descriptor instead.
+func (*ReleaseLeaseRequest) Descriptor() ([]byte, []int) {
+	return file_pkg_distlock_grpc_lease_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *ReleaseLeaseRequest) GetResource() string {
+	if x != nil {
+		return x.Resource
+	}
+	return ""
+}
+
+func (x *ReleaseLeaseRequest) GetToken() []byte {
+	if x != nil {
+		return x.Token
+	}
+	return nil
+}
+
+type ReleaseLeaseResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Released is true when an unexpired lease existed, matched the ownership token, and was successfully released.
+	Released bool `protobuf:"varint,1,opt,name=released,proto3" json:"released,omitempty"`
+}
+
+func (x *ReleaseLeaseResponse) Reset() {
+	*x = ReleaseLeaseResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ReleaseLeaseResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ReleaseLeaseResponse) ProtoMessage() {}
+
+func (x *ReleaseLeaseResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_distlock_grpc_lease_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ReleaseLeaseResponse.ProtoReflect.Descriptor instead.
+func (*ReleaseLeaseResponse) Descriptor() ([]byte, []int) {
+	return file_pkg_distlock_grpc_lease_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *ReleaseLeaseResponse) GetReleased() bool {
+	if x != nil {
+		return x.Released
+	}
+	return false
+}
+
+var File_pkg_distlock_grpc_lease_proto protoreflect.FileDescriptor
+
+var file_pkg_distlock_grpc_lease_proto_rawDesc = []byte{
+	0x0a, 0x1d, 0x70, 0x6b, 0x67, 0x2f, 0x64, 0x69, 0x73, 0x74, 0x6c, 0x6f, 0x63, 0x6b, 0x2f, 0x67,
+	0x72, 0x70, 0x63, 0x2f, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
+	0x0b, 0x64, 0x69, 0x73, 0x74, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x76, 0x31, 0x1a, 0x1e, 0x67, 0x6f,
+	0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75,
+	0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x74, 0x0a, 0x13,
+	0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12,
+	0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05,
+	0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x74,
+	0x74, 0x6c, 0x22, 0x32, 0x0a, 0x14, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61,
+	0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x63,
+	0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x61, 0x63,
+	0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x22, 0x72, 0x0a, 0x11, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x4c,
+	0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72,
+	0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72,
+	0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b, 0x0a,
+	0x03, 0x74, 0x74, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f,
+	0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x22, 0x2e, 0x0a, 0x12, 0x52, 0x65,
+	0x6e, 0x65, 0x77, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x08, 0x52, 0x07, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x65, 0x64, 0x22, 0x47, 0x0a, 0x13, 0x52, 0x65,
+	0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x14, 0x0a,
+	0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x74, 0x6f,
+	0x6b, 0x65, 0x6e, 0x22, 0x32, 0x0a, 0x14, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c, 0x65,
+	0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72,
+	0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72,
+	0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x64, 0x32, 0xf1, 0x01, 0x0a, 0x05, 0x4c, 0x65, 0x61, 0x73,
+	0x65, 0x12, 0x4e, 0x0a, 0x07, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x12, 0x20, 0x2e, 0x64,
+	0x69, 0x73, 0x74, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69,
+	0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21,
+	0x2e, 0x64, 0x69, 0x73, 0x74, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x71,
+	0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x12, 0x48, 0x0a, 0x05, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x12, 0x1e, 0x2e, 0x64, 0x69, 0x73,
+	0x74, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65,
+	0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x64, 0x69, 0x73,
+	0x74, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65,
+	0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x07, 0x52,
+	0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x20, 0x2e, 0x64, 0x69, 0x73, 0x74, 0x6c, 0x6f, 0x63,
+	0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c, 0x65, 0x61, 0x73,
+	0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x64, 0x69, 0x73, 0x74, 0x6c,
+	0x6f, 0x63, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c, 0x65,
+	0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x31, 0x5a, 0x2f, 0x67,
+	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65,
+	0x72, 0x73, 0x6b, 0x69, 0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x70, 0x6b, 0x67,
+	0x2f, 0x64, 0x69, 0x73, 0x74, 0x6c, 0x6f, 0x63, 0x6b, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_pkg_distlock_grpc_lease_proto_rawDescOnce sync.Once
+	file_pkg_distlock_grpc_lease_proto_rawDescData = file_pkg_distlock_grpc_lease_proto_rawDesc
+)
+
+func file_pkg_distlock_grpc_lease_proto_rawDescGZIP() []byte {
+	file_pkg_distlock_grpc_lease_proto_rawDescOnce.Do(func() {
+		file_pkg_distlock_grpc_lease_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_distlock_grpc_lease_proto_rawDescData)
+	})
+	return file_pkg_distlock_grpc_lease_proto_rawDescData
+}
+
+var file_pkg_distlock_grpc_lease_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
+var file_pkg_distlock_grpc_lease_proto_goTypes = []any{
+	(*AcquireLeaseRequest)(nil),  // 0: distlock.v1.AcquireLeaseRequest
+	(*AcquireLeaseResponse)(nil), // 1: distlock.v1.AcquireLeaseResponse
+	(*RenewLeaseRequest)(nil),    // 2: distlock.v1.RenewLeaseRequest
+	(*RenewLeaseResponse)(nil),   // 3: distlock.v1.RenewLeaseResponse
+	(*ReleaseLeaseRequest)(nil),  // 4: distlock.v1.ReleaseLeaseRequest
+	(*ReleaseLeaseResponse)(nil), // 5: distlock.v1.ReleaseLeaseResponse
+	(*durationpb.Duration)(nil),  // 6: google.protobuf.Duration
+}
+var file_pkg_distlock_grpc_lease_proto_depIdxs = []int32{
+	6, // 0: distlock.v1.AcquireLeaseRequest.ttl:type_name -> google.protobuf.Duration
+	6, // 1: distlock.v1.RenewLeaseRequest.ttl:type_name -> google.protobuf.Duration
+	0, // 2: distlock.v1.Lease.Acquire:input_type -> distlock.v1.AcquireLeaseRequest
+	2, // 3: distlock.v1.Lease.Renew:input_type -> distlock.v1.RenewLeaseRequest
+	4, // 4: distlock.v1.Lease.Release:input_type -> distlock.v1.ReleaseLeaseRequest
+	1, // 5: distlock.v1.Lease.Acquire:output_type -> distlock.v1.AcquireLeaseResponse
+	3, // 6: distlock.v1.Lease.Renew:output_type -> distlock.v1.RenewLeaseResponse
+	5, // 7: distlock.v1.Lease.Release:output_type -> distlock.v1.ReleaseLeaseResponse
+	5, // [5:8] is the sub-list for method output_type
+	2, // [2:5] is the sub-list for method input_type
+	2, // [2:2] is the sub-list for extension type_name
+	2, // [2:2] is the sub-list for extension extendee
+	0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_pkg_distlock_grpc_lease_proto_init() }
+func file_pkg_distlock_grpc_lease_proto_init() {
+	if File_pkg_distlock_grpc_lease_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_pkg_distlock_grpc_lease_proto_msgTypes[0].Exporter = func(v any, i int) any {
+			switch v := v.(*AcquireLeaseRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkg_distlock_grpc_lease_proto_msgTypes[1].Exporter = func(v any, i int) any {
+			switch v := v.(*AcquireLeaseResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkg_distlock_grpc_lease_proto_msgTypes[2].Exporter = func(v any, i int) any {
+			switch v := v.(*RenewLeaseRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkg_distlock_grpc_lease_proto_msgTypes[3].Exporter = func(v any, i int) any {
+			switch v := v.(*RenewLeaseResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkg_distlock_grpc_lease_proto_msgTypes[4].Exporter = func(v any, i int) any {
+			switch v := v.(*ReleaseLeaseRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkg_distlock_grpc_lease_proto_msgTypes[5].Exporter = func(v any, i int) any {
+			switch v := v.(*ReleaseLeaseResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_pkg_distlock_grpc_lease_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   6,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_pkg_distlock_grpc_lease_proto_goTypes,
+		DependencyIndexes: file_pkg_distlock_grpc_lease_proto_depIdxs,
+		MessageInfos:      file_pkg_distlock_grpc_lease_proto_msgTypes,
+	}.Build()
+	File_pkg_distlock_grpc_lease_proto = out.File
+	file_pkg_distlock_grpc_lease_proto_rawDesc = nil
+	file_pkg_distlock_grpc_lease_proto_goTypes = nil
+	file_pkg_distlock_grpc_lease_proto_depIdxs = nil
+}

pkg/distlock/grpc/lease.proto

@@ -0,0 +1,54 @@
+syntax = "proto3";
+
+package distlock.v1;
+
+option go_package = "github.com/psviderski/uncloud/pkg/distlock/grpc";
+
+import "google/protobuf/duration.proto";
+
+// Lease provides atomic operations for time-bound ownership of resources on one node.
+service Lease {
+  // Acquire creates a lease when the resource has no unexpired lease.
+  rpc Acquire(AcquireLeaseRequest) returns (AcquireLeaseResponse);
+  // Renew extends an unexpired lease when its ownership token matches.
+  rpc Renew(RenewLeaseRequest) returns (RenewLeaseResponse);
+  // Release removes an unexpired lease when its ownership token matches.
+  rpc Release(ReleaseLeaseRequest) returns (ReleaseLeaseResponse);
+}
+
+message AcquireLeaseRequest {
+  string resource = 1;
+  // Token uniquely identifies the lease owner.
+  bytes token = 2;
+  // TTL sets how long the lease remains valid without renewal.
+  google.protobuf.Duration ttl = 3;
+}
+
+message AcquireLeaseResponse {
+  // Acquired is true when this request successfully created the lease.
+  bool acquired = 1;
+}
+
+message RenewLeaseRequest {
+  string resource = 1;
+  // Token identifies the owner of the existing lease.
+  bytes token = 2;
+  // TTL sets how long the renewed lease remains valid.
+  google.protobuf.Duration ttl = 3;
+}
+
+message RenewLeaseResponse {
+  // Renewed is true when an unexpired lease matched the ownership token and was successfully renewed.
+  bool renewed = 1;
+}
+
+message ReleaseLeaseRequest {
+  string resource = 1;
+  // Token identifies the owner of the existing lease.
+  bytes token = 2;
+}
+
+message ReleaseLeaseResponse {
+  // Released is true when an unexpired lease existed, matched the ownership token, and was successfully released.
+  bool released = 1;
+}

pkg/distlock/grpc/lease_grpc.pb.go

@@ -0,0 +1,207 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v5.27.3
+// source: pkg/distlock/grpc/lease.proto
+
+package grpc
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	Lease_Acquire_FullMethodName = "/distlock.v1.Lease/Acquire"
+	Lease_Renew_FullMethodName   = "/distlock.v1.Lease/Renew"
+	Lease_Release_FullMethodName = "/distlock.v1.Lease/Release"
+)
+
+// LeaseClient is the client API for Lease service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+//
+// Lease provides atomic operations for time-bound ownership of resources on one node.
+type LeaseClient interface {
+	// Acquire creates a lease when the resource has no unexpired lease.
+	Acquire(ctx context.Context, in *AcquireLeaseRequest, opts ...grpc.CallOption) (*AcquireLeaseResponse, error)
+	// Renew extends an unexpired lease when its ownership token matches.
+	Renew(ctx context.Context, in *RenewLeaseRequest, opts ...grpc.CallOption) (*RenewLeaseResponse, error)
+	// Release removes an unexpired lease when its ownership token matches.
+	Release(ctx context.Context, in *ReleaseLeaseRequest, opts ...grpc.CallOption) (*ReleaseLeaseResponse, error)
+}
+
+type leaseClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewLeaseClient(cc grpc.ClientConnInterface) LeaseClient {
+	return &leaseClient{cc}
+}
+
+func (c *leaseClient) Acquire(ctx context.Context, in *AcquireLeaseRequest, opts ...grpc.CallOption) (*AcquireLeaseResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(AcquireLeaseResponse)
+	err := c.cc.Invoke(ctx, Lease_Acquire_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *leaseClient) Renew(ctx context.Context, in *RenewLeaseRequest, opts ...grpc.CallOption) (*RenewLeaseResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(RenewLeaseResponse)
+	err := c.cc.Invoke(ctx, Lease_Renew_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *leaseClient) Release(ctx context.Context, in *ReleaseLeaseRequest, opts ...grpc.CallOption) (*ReleaseLeaseResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ReleaseLeaseResponse)
+	err := c.cc.Invoke(ctx, Lease_Release_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// LeaseServer is the server API for Lease service.
+// All implementations must embed UnimplementedLeaseServer
+// for forward compatibility.
+//
+// Lease provides atomic operations for time-bound ownership of resources on one node.
+type LeaseServer interface {
+	// Acquire creates a lease when the resource has no unexpired lease.
+	Acquire(context.Context, *AcquireLeaseRequest) (*AcquireLeaseResponse, error)
+	// Renew extends an unexpired lease when its ownership token matches.
+	Renew(context.Context, *RenewLeaseRequest) (*RenewLeaseResponse, error)
+	// Release removes an unexpired lease when its ownership token matches.
+	Release(context.Context, *ReleaseLeaseRequest) (*ReleaseLeaseResponse, error)
+	mustEmbedUnimplementedLeaseServer()
+}
+
+// UnimplementedLeaseServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedLeaseServer struct{}
+
+func (UnimplementedLeaseServer) Acquire(context.Context, *AcquireLeaseRequest) (*AcquireLeaseResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Acquire not implemented")
+}
+func (UnimplementedLeaseServer) Renew(context.Context, *RenewLeaseRequest) (*RenewLeaseResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Renew not implemented")
+}
+func (UnimplementedLeaseServer) Release(context.Context, *ReleaseLeaseRequest) (*ReleaseLeaseResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Release not implemented")
+}
+func (UnimplementedLeaseServer) mustEmbedUnimplementedLeaseServer() {}
+func (UnimplementedLeaseServer) testEmbeddedByValue()               {}
+
+// UnsafeLeaseServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to LeaseServer will
+// result in compilation errors.
+type UnsafeLeaseServer interface {
+	mustEmbedUnimplementedLeaseServer()
+}
+
+func RegisterLeaseServer(s grpc.ServiceRegistrar, srv LeaseServer) {
+	// If the following call pancis, it indicates UnimplementedLeaseServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&Lease_ServiceDesc, srv)
+}
+
+func _Lease_Acquire_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(AcquireLeaseRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(LeaseServer).Acquire(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: Lease_Acquire_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(LeaseServer).Acquire(ctx, req.(*AcquireLeaseRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Lease_Renew_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(RenewLeaseRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(LeaseServer).Renew(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: Lease_Renew_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(LeaseServer).Renew(ctx, req.(*RenewLeaseRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Lease_Release_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ReleaseLeaseRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(LeaseServer).Release(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: Lease_Release_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(LeaseServer).Release(ctx, req.(*ReleaseLeaseRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// Lease_ServiceDesc is the grpc.ServiceDesc for Lease service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var Lease_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "distlock.v1.Lease",
+	HandlerType: (*LeaseServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Acquire",
+			Handler:    _Lease_Acquire_Handler,
+		},
+		{
+			MethodName: "Renew",
+			Handler:    _Lease_Renew_Handler,
+		},
+		{
+			MethodName: "Release",
+			Handler:    _Lease_Release_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "pkg/distlock/grpc/lease.proto",
+}

pkg/distlock/grpc/server.go

@@ -0,0 +1,99 @@
+// Package grpc provides a gRPC transport for distlock node-local lease operations.
+package grpc
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/psviderski/uncloud/pkg/distlock"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/durationpb"
+)
+
+// Server adapts a node-local distlock.Store to the Lease gRPC service.
+type Server struct {
+	UnimplementedLeaseServer
+	store distlock.Store
+}
+
+// NewServer creates a node-local lease server.
+func NewServer(store distlock.Store) *Server {
+	return &Server{store: store}
+}
+
+// Acquire creates a lease when the resource has no unexpired lease.
+func (s *Server) Acquire(ctx context.Context, req *AcquireLeaseRequest) (*AcquireLeaseResponse, error) {
+	ttl, err := validateLeaseRequest(req.Resource, req.Token, req.Ttl)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	acquired, err := s.store.Acquire(ctx, req.Resource, req.Token, ttl)
+	if err != nil {
+		return nil, storeStatusError(ctx, "acquire lease", err)
+	}
+	return &AcquireLeaseResponse{Acquired: acquired}, nil
+}
+
+// Renew extends an unexpired lease when its ownership token matches.
+func (s *Server) Renew(ctx context.Context, req *RenewLeaseRequest) (*RenewLeaseResponse, error) {
+	ttl, err := validateLeaseRequest(req.Resource, req.Token, req.Ttl)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	renewed, err := s.store.Renew(ctx, req.Resource, req.Token, ttl)
+	if err != nil {
+		return nil, storeStatusError(ctx, "renew lease", err)
+	}
+	return &RenewLeaseResponse{Renewed: renewed}, nil
+}
+
+// Release removes an unexpired lease when its ownership token matches.
+func (s *Server) Release(ctx context.Context, req *ReleaseLeaseRequest) (*ReleaseLeaseResponse, error) {
+	if err := validateResourceToken(req.Resource, req.Token); err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	released, err := s.store.Release(ctx, req.Resource, req.Token)
+	if err != nil {
+		return nil, storeStatusError(ctx, "release lease", err)
+	}
+	return &ReleaseLeaseResponse{Released: released}, nil
+}
+
+func validateLeaseRequest(resource string, token []byte, ttl *durationpb.Duration) (time.Duration, error) {
+	if err := validateResourceToken(resource, token); err != nil {
+		return 0, err
+	}
+	if ttl == nil {
+		return 0, fmt.Errorf("TTL is not set")
+	}
+	if err := ttl.CheckValid(); err != nil {
+		return 0, fmt.Errorf("invalid TTL: %w", err)
+	}
+	duration := ttl.AsDuration()
+	if duration <= 0 {
+		return 0, fmt.Errorf("TTL must be positive")
+	}
+	return duration, nil
+}
+
+func validateResourceToken(resource string, token []byte) error {
+	if resource == "" {
+		return fmt.Errorf("resource is empty")
+	}
+	if len(token) == 0 {
+		return fmt.Errorf("token is empty")
+	}
+	return nil
+}
+
+func storeStatusError(ctx context.Context, operation string, err error) error {
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return status.FromContextError(ctxErr).Err()
+	}
+	return status.Error(codes.Internal, fmt.Sprintf("%s: %v", operation, err))
+}

pkg/distlock/locker.go

@@ -0,0 +1,393 @@
+package distlock
+
+import (
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"math"
+	"slices"
+	"sync"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+)
+
+const (
+	DefaultLeaseDuration      = 10 * time.Second
+	DefaultClockDriftFactor   = 0.01
+	DefaultMaxNodeCallTimeout = 5 * time.Second
+)
+
+var (
+	// ErrLeaseLost is the cancellation cause when automatic renewal can no longer maintain a lease.
+	ErrLeaseLost = errors.New("distributed lease lost")
+	// ErrLeaseReleased is the cancellation cause of an explicitly released lease.
+	ErrLeaseReleased = errors.New("distributed lease released")
+)
+
+// Config configures a Locker.
+type Config struct {
+	// LeaseDuration is the TTL used for acquisitions and renewals. The default is DefaultLeaseDuration.
+	LeaseDuration time.Duration
+	// ClockDriftFactor is the fraction of LeaseDuration reserved for differences in clock rates between the Locker and
+	// nodes. The default is DefaultClockDriftFactor.
+	ClockDriftFactor float64
+	// NodeCallTimeout sets the context timeout for an Acquire, Renew, or Release call to one node.
+	// The default is the smaller of DefaultMaxNodeCallTimeout and one third of the lease duration.
+	NodeCallTimeout time.Duration
+	// NewBackOff creates independent retry policies for acquisitions and renewal cycles. The default is an exponential
+	// backoff starting at 100ms and capped at 1s. A policy should not impose its own elapsed-time limit because the
+	// acquisition context and current lease validity already bound retries.
+	NewBackOff func() backoff.BackOff
+}
+
+func (c Config) clockDrift() time.Duration {
+	return time.Duration(math.Ceil(float64(c.LeaseDuration) * c.ClockDriftFactor))
+}
+
+// Locker acquires automatically renewed distributed leases over a Cluster.
+type Locker struct {
+	config  Config
+	cluster Cluster
+}
+
+// New creates a Locker over cluster.
+func New(cluster Cluster, config Config) (*Locker, error) {
+	if cluster == nil {
+		return nil, fmt.Errorf("cluster is nil")
+	}
+
+	if config.LeaseDuration == 0 {
+		config.LeaseDuration = DefaultLeaseDuration
+	}
+	if config.LeaseDuration < 0 {
+		return nil, fmt.Errorf("lease duration must be positive")
+	}
+	if config.ClockDriftFactor == 0 {
+		config.ClockDriftFactor = DefaultClockDriftFactor
+	}
+	if config.ClockDriftFactor <= 0 || config.ClockDriftFactor >= 1 {
+		return nil, fmt.Errorf("clock drift factor must be greater than 0 and less than 1")
+	}
+	if config.NodeCallTimeout == 0 {
+		config.NodeCallTimeout = min(DefaultMaxNodeCallTimeout, config.LeaseDuration/3)
+	}
+	if config.NodeCallTimeout < 0 {
+		return nil, fmt.Errorf("node call timeout must be positive")
+	}
+	if config.NewBackOff == nil {
+		config.NewBackOff = defaultBackOff
+	}
+
+	return &Locker{config: config, cluster: cluster}, nil
+}
+
+func defaultBackOff() backoff.BackOff {
+	return backoff.NewExponentialBackOff(
+		backoff.WithInitialInterval(100*time.Millisecond),
+		backoff.WithMaxInterval(time.Second),
+		backoff.WithMaxElapsedTime(0),
+	)
+}
+
+// Acquire waits until it acquires a lease for resource or ctx ends.
+func (l *Locker) Acquire(ctx context.Context, resource string) (*Lease, error) {
+	if resource == "" {
+		return nil, fmt.Errorf("resource is empty")
+	}
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+
+	nodes, err := l.cluster.Nodes(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("get cluster nodes: %w", err)
+	}
+	if len(nodes) == 0 {
+		return nil, fmt.Errorf("cluster has no nodes")
+	}
+	nodes = slices.Clone(nodes)
+
+	boff := backoff.WithContext(l.config.NewBackOff(), ctx)
+	var lease *Lease
+	err = backoff.Retry(func() error {
+		token, tokenErr := newOwnershipToken()
+		if tokenErr != nil {
+			return backoff.Permanent(fmt.Errorf("generate lease token: %w", tokenErr))
+		}
+
+		candidate := newLease(l, nodes, resource, token)
+		if err := candidate.acquire(ctx); err != nil {
+			return err
+		}
+
+		lease = candidate
+		return nil
+	}, boff)
+	if err != nil {
+		return nil, fmt.Errorf("acquire distributed lease for %q: %w", resource, err)
+	}
+
+	return lease, nil
+}
+
+// newOwnershipToken generates a unique 128-bit random ownership token for a lease.
+func newOwnershipToken() ([]byte, error) {
+	token := make([]byte, 16)
+	if _, err := rand.Read(token); err != nil {
+		return nil, err
+	}
+	return token, nil
+}
+
+// Lease is an automatically renewed distributed lease.
+type Lease struct {
+	config   Config
+	nodes    []Node
+	resource string
+	token    []byte
+
+	ctx    context.Context
+	cancel context.CancelCauseFunc
+	// done is closed when the renewal goroutine exits.
+	done chan struct{}
+
+	// operationMu prevents acquisition, renewal, and release operations for the lease from overlapping.
+	operationMu sync.Mutex
+	quorum      int
+}
+
+func newLease(locker *Locker, nodes []Node, resource string, token []byte) *Lease {
+	ctx, cancel := context.WithCancelCause(context.Background())
+	return &Lease{
+		config:   locker.config,
+		nodes:    nodes,
+		resource: resource,
+		token:    token,
+		ctx:      ctx,
+		cancel:   cancel,
+		done:     make(chan struct{}),
+		quorum:   len(nodes)/2 + 1,
+	}
+}
+
+// Context returns a context that is cancelled when the lease is lost or explicitly released.
+func (l *Lease) Context() context.Context {
+	return l.ctx
+}
+
+// Release stops automatic renewal and attempts to remove the lease from every node in its acquisition snapshot.
+func (l *Lease) Release(ctx context.Context) error {
+	l.cancel(ErrLeaseReleased)
+	select {
+	case <-l.done:
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	if err := l.release(ctx); err != nil {
+		return fmt.Errorf("release distributed lease for %q: %w", l.resource, err)
+	}
+	return nil
+}
+
+type nodeResult struct {
+	success bool
+	err     error
+}
+
+func collectNodeResults(results <-chan nodeResult) (successes int, err error) {
+	var errs []error
+	for result := range results {
+		if result.err != nil {
+			errs = append(errs, result.err)
+		} else if result.success {
+			successes++
+		}
+	}
+	return successes, errors.Join(errs...)
+}
+
+func (l *Lease) executeNodes(ctx context.Context, fn func(context.Context, Node) (bool, error)) <-chan nodeResult {
+	results := make(chan nodeResult, len(l.nodes))
+	var wg sync.WaitGroup
+	for _, node := range l.nodes {
+		wg.Go(func() {
+			callCtx, cancel := context.WithTimeout(ctx, l.config.NodeCallTimeout)
+			defer cancel()
+
+			success, err := fn(callCtx, node)
+			results <- nodeResult{success: success, err: err}
+		})
+	}
+	go func() {
+		wg.Wait()
+		close(results)
+	}()
+	return results
+}
+
+// acquire makes one attempt to obtain the lease from a quorum of nodes and starts renewal on success.
+func (l *Lease) acquire(ctx context.Context) error {
+	startedAt := time.Now()
+	validUntil := startedAt.Add(l.config.LeaseDuration - l.config.clockDrift())
+	resultCh := make(chan error, 1)
+	// Aggregate asynchronously so acquire can return at quorum while this goroutine drains the remaining results and
+	// holds operationMu until every node call has finished.
+	go func() {
+		l.operationMu.Lock()
+		defer l.operationMu.Unlock()
+
+		// Use the lease context so cancelling the context passed to Acquire after it succeeds does not stop node calls
+		// still pending after quorum. On failure, acquire cancels the lease context below. The validity deadline and
+		// per-node NodeCallTimeout bound these calls.
+		operationCtx, cancel := context.WithDeadline(l.ctx, validUntil)
+		defer cancel()
+		results := l.executeNodes(operationCtx, func(ctx context.Context, node Node) (bool, error) {
+			return node.Acquire(ctx, l.resource, l.token, l.config.LeaseDuration)
+		})
+
+		successes := 0
+		errs := make([]error, 0, len(l.nodes))
+		reported := false
+		for result := range results {
+			if result.err != nil {
+				errs = append(errs, result.err)
+			} else if result.success {
+				successes++
+			}
+			if !reported && successes >= l.quorum {
+				resultCh <- nil
+				reported = true
+			}
+		}
+		if !reported {
+			quorumErr := fmt.Errorf("lease acquired on %d of %d nodes, need at least %d",
+				successes, len(l.nodes), l.quorum)
+			resultCh <- errors.Join(quorumErr, errors.Join(errs...))
+		}
+	}()
+
+	var acquireErr error
+	select {
+	case acquireErr = <-resultCh:
+	case <-ctx.Done():
+		acquireErr = ctx.Err()
+	}
+
+	if acquireErr == nil {
+		if !time.Now().Before(validUntil) {
+			acquireErr = fmt.Errorf("lease validity expired during acquisition")
+		} else {
+			go l.runRenew(validUntil)
+			return nil
+		}
+	}
+
+	// Cancel any node Acquire calls still in progress. release waits for them to finish before removing partial leases,
+	// so no node can create this lease after cleanup has run.
+	l.cancel(acquireErr)
+	_ = l.release(context.WithoutCancel(ctx))
+	return acquireErr
+}
+
+// release waits for any in-progress lease operation, removes the lease from every node, and returns any errors.
+func (l *Lease) release(ctx context.Context) error {
+	l.operationMu.Lock()
+	defer l.operationMu.Unlock()
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	results := l.executeNodes(ctx, func(ctx context.Context, node Node) (bool, error) {
+		return node.Release(ctx, l.resource, l.token)
+	})
+	_, err := collectNodeResults(results)
+	return err
+}
+
+// runRenew periodically renews the lease until it is released or lost.
+func (l *Lease) runRenew(validUntil time.Time) {
+	defer close(l.done)
+
+	for {
+		remaining := time.Until(validUntil)
+		if remaining <= 0 {
+			break
+		}
+
+		// Start renewal with two thirds of the current validity remaining
+		// to leave time for slow node calls and retries.
+		timer := time.NewTimer(remaining / 3)
+		select {
+		case <-l.ctx.Done():
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+
+		renewedUntil, err := l.renew(validUntil)
+		if err != nil {
+			if l.ctx.Err() != nil {
+				return
+			}
+			break
+		}
+		validUntil = renewedUntil
+	}
+
+	l.cancel(ErrLeaseLost)
+	_ = l.release(context.Background())
+}
+
+// renew retries node renewals until a quorum succeeds, the configured backoff stops, or the current lease validity
+// ends. It returns the new validity deadline after reaching quorum.
+func (l *Lease) renew(currentValidUntil time.Time) (time.Time, error) {
+	ctx, cancel := context.WithDeadline(l.ctx, currentValidUntil)
+	defer cancel()
+
+	var validUntil time.Time
+	resultCh := make(chan error, 1)
+	// Coordinate in the background so renew can report expiry even if a node call does not return after cancellation.
+	// Keep operationMu held until every call finishes so cleanup cannot race a pending renewal.
+	go func() {
+		l.operationMu.Lock()
+		defer l.operationMu.Unlock()
+
+		if err := ctx.Err(); err != nil {
+			resultCh <- err
+			return
+		}
+
+		boff := backoff.WithContext(l.config.NewBackOff(), ctx)
+		resultCh <- backoff.Retry(func() error {
+			startedAt := time.Now()
+			results := l.executeNodes(ctx, func(ctx context.Context, node Node) (bool, error) {
+				return node.Renew(ctx, l.resource, l.token, l.config.LeaseDuration)
+			})
+			successes, nodeErr := collectNodeResults(results)
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+			if !time.Now().Before(currentValidUntil) {
+				return backoff.Permanent(fmt.Errorf("renewal attempt took longer than its validity window"))
+			}
+
+			if successes >= l.quorum {
+				validUntil = startedAt.Add(l.config.LeaseDuration - l.config.clockDrift())
+				return nil
+			}
+			quorumErr := fmt.Errorf("lease renewed on %d of %d nodes, need at least %d",
+				successes, len(l.nodes), l.quorum)
+			return errors.Join(quorumErr, nodeErr)
+		}, boff)
+	}()
+
+	select {
+	case err := <-resultCh:
+		return validUntil, err
+	case <-ctx.Done():
+		return time.Time{}, ctx.Err()
+	}
+}

pkg/distlock/locker_test.go

@@ -0,0 +1,445 @@
+package distlock_test
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/psviderski/uncloud/pkg/distlock"
+	"github.com/stretchr/testify/require"
+)
+
+var errMemoryNodeUnavailable = errors.New("memory node unavailable")
+
+type memoryCluster struct {
+	nodes []*memoryNode
+}
+
+func newMemoryCluster(size int) *memoryCluster {
+	cluster := &memoryCluster{nodes: make([]*memoryNode, size)}
+	for i := range cluster.nodes {
+		cluster.nodes[i] = &memoryNode{store: distlock.NewMemoryStore(), available: true}
+	}
+	return cluster
+}
+
+func (c *memoryCluster) Nodes(ctx context.Context) ([]distlock.Node, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+
+	nodes := make([]distlock.Node, len(c.nodes))
+	for i, node := range c.nodes {
+		nodes[i] = node
+	}
+	return nodes, nil
+}
+
+type memoryNode struct {
+	mu                  sync.Mutex
+	store               distlock.Store
+	available           bool
+	unavailableObserved chan struct{}
+}
+
+func (n *memoryNode) setAvailable(available bool) {
+	if !available {
+		n.makeUnavailable()
+		return
+	}
+
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	n.available = true
+}
+
+func (n *memoryNode) makeUnavailable() <-chan struct{} {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	n.available = false
+	n.unavailableObserved = make(chan struct{})
+	return n.unavailableObserved
+}
+
+func (n *memoryNode) restart() {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	n.store = distlock.NewMemoryStore()
+}
+
+func (n *memoryNode) currentStore(ctx context.Context) (distlock.Store, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if !n.available {
+		if n.unavailableObserved != nil {
+			close(n.unavailableObserved)
+			n.unavailableObserved = nil
+		}
+		return nil, errMemoryNodeUnavailable
+	}
+	return n.store, nil
+}
+
+func (n *memoryNode) Acquire(
+	ctx context.Context, resource string, token []byte, ttl time.Duration,
+) (bool, error) {
+	store, err := n.currentStore(ctx)
+	if err != nil {
+		return false, err
+	}
+	return store.Acquire(ctx, resource, token, ttl)
+}
+
+func (n *memoryNode) Renew(
+	ctx context.Context, resource string, token []byte, ttl time.Duration,
+) (bool, error) {
+	store, err := n.currentStore(ctx)
+	if err != nil {
+		return false, err
+	}
+	return store.Renew(ctx, resource, token, ttl)
+}
+
+func (n *memoryNode) Release(ctx context.Context, resource string, token []byte) (bool, error) {
+	store, err := n.currentStore(ctx)
+	if err != nil {
+		return false, err
+	}
+	return store.Release(ctx, resource, token)
+}
+
+func retryBackOff() backoff.BackOff {
+	return backoff.NewConstantBackOff(5 * time.Millisecond)
+}
+
+func oneAttemptBackOff() backoff.BackOff {
+	return &backoff.StopBackOff{}
+}
+
+func newTestLocker(
+	t *testing.T, cluster distlock.Cluster, leaseDuration time.Duration, newBackOff func() backoff.BackOff,
+) *distlock.Locker {
+	t.Helper()
+	locker, err := distlock.New(cluster, distlock.Config{
+		LeaseDuration:   leaseDuration,
+		NodeCallTimeout: leaseDuration / 3,
+		NewBackOff:      newBackOff,
+	})
+	require.NoError(t, err)
+	return locker
+}
+
+func acquireLease(t *testing.T, locker *distlock.Locker, resource string) *distlock.Lease {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	lease, err := locker.Acquire(ctx, resource)
+	require.NoError(t, err)
+	return lease
+}
+
+func releaseLease(t *testing.T, lease *distlock.Lease) {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	require.NoError(t, lease.Release(ctx))
+}
+
+func requireNoReceive[T any](t *testing.T, ch <-chan T, timeout time.Duration, message string) {
+	t.Helper()
+	select {
+	case <-ch:
+		require.FailNow(t, message)
+	case <-time.After(timeout):
+	}
+}
+
+func requireReceive[T any](t *testing.T, ch <-chan T, timeout time.Duration, message string) T {
+	t.Helper()
+	select {
+	case value := <-ch:
+		return value
+	case <-time.After(timeout):
+		require.FailNow(t, message)
+		var zero T
+		return zero
+	}
+}
+
+func requireContextActive(t *testing.T, ctx context.Context, message string) {
+	t.Helper()
+	select {
+	case <-ctx.Done():
+		require.FailNow(t, message, "cause: %v", context.Cause(ctx))
+	default:
+	}
+}
+
+func requireSignal(t *testing.T, ch <-chan struct{}, timeout time.Duration, message string) {
+	t.Helper()
+	select {
+	case <-ch:
+	case <-time.After(timeout):
+		require.FailNow(t, message)
+	}
+}
+
+func TestLockerAcquireQuorumBoundaries(t *testing.T) {
+	tests := []struct {
+		name        string
+		nodes       int
+		unavailable int
+		wantAcquire bool
+	}{
+		{name: "two nodes at quorum", nodes: 2, wantAcquire: true},
+		{name: "two nodes below quorum", nodes: 2, unavailable: 1, wantAcquire: false},
+		{name: "three nodes at quorum", nodes: 3, unavailable: 1, wantAcquire: true},
+		{name: "three nodes below quorum", nodes: 3, unavailable: 2, wantAcquire: false},
+		{name: "four nodes at quorum", nodes: 4, unavailable: 1, wantAcquire: true},
+		{name: "four nodes below quorum", nodes: 4, unavailable: 2, wantAcquire: false},
+		{name: "five nodes at quorum", nodes: 5, unavailable: 2, wantAcquire: true},
+		{name: "five nodes below quorum", nodes: 5, unavailable: 3, wantAcquire: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cluster := newMemoryCluster(tt.nodes)
+			unavailableObserved := make([]<-chan struct{}, 0, tt.unavailable)
+			for i := range tt.unavailable {
+				unavailableObserved = append(unavailableObserved, cluster.nodes[tt.nodes-1-i].makeUnavailable())
+			}
+			locker := newTestLocker(t, cluster, 300*time.Millisecond, oneAttemptBackOff)
+
+			acquireCtx, cancelAcquire := context.WithTimeout(context.Background(), 2*time.Second)
+			lease, err := locker.Acquire(acquireCtx, "resource")
+			cancelAcquire()
+			for _, observed := range unavailableObserved {
+				requireSignal(t, observed, time.Second, "unavailable node did not receive acquisition")
+			}
+			for i := range tt.unavailable {
+				cluster.nodes[tt.nodes-1-i].setAvailable(true)
+			}
+
+			if tt.wantAcquire {
+				require.NoError(t, err)
+				require.NotNil(t, lease)
+				releaseLease(t, lease)
+			} else {
+				if lease != nil {
+					releaseLease(t, lease)
+				}
+				require.Error(t, err)
+				require.Nil(t, lease)
+			}
+		})
+	}
+}
+
+func TestLockerSingleNodeLifecycle(t *testing.T) {
+	const leaseDuration = 300 * time.Millisecond
+	cluster := newMemoryCluster(1)
+	locker := newTestLocker(t, cluster, leaseDuration, oneAttemptBackOff)
+
+	lease := acquireLease(t, locker, "resource")
+	defer func() {
+		if lease.Context().Err() != nil {
+			return
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		_ = lease.Release(ctx)
+	}()
+
+	time.Sleep(leaseDuration + 100*time.Millisecond)
+	requireContextActive(t, lease.Context(), "single-node lease was not renewed")
+	releaseLease(t, lease)
+
+	secondLease := acquireLease(t, locker, "resource")
+	releaseLease(t, secondLease)
+}
+
+func TestLockerFailedAcquireCleansUpPartialLease(t *testing.T) {
+	cluster := newMemoryCluster(3)
+	cluster.nodes[1].setAvailable(false)
+	cluster.nodes[2].setAvailable(false)
+	locker := newTestLocker(t, cluster, 300*time.Millisecond, oneAttemptBackOff)
+
+	lease, err := locker.Acquire(context.Background(), "resource")
+	require.Error(t, err)
+	require.Nil(t, lease)
+
+	cluster.nodes[1].setAvailable(true)
+	cluster.nodes[2].setAvailable(true)
+	lease = acquireLease(t, locker, "resource")
+	releaseLease(t, lease)
+}
+
+func TestLockerAcquiresIndependentResourcesConcurrently(t *testing.T) {
+	cluster := newMemoryCluster(3)
+	locker := newTestLocker(t, cluster, 300*time.Millisecond, oneAttemptBackOff)
+	resources := []string{"database", "deployment", "network", "volume"}
+
+	type acquireResult struct {
+		resource string
+		lease    *distlock.Lease
+		err      error
+	}
+	resultCh := make(chan acquireResult, len(resources))
+	start := make(chan struct{})
+	acquireCtx, cancelAcquire := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancelAcquire()
+	for _, resource := range resources {
+		go func() {
+			<-start
+			lease, err := locker.Acquire(acquireCtx, resource)
+			resultCh <- acquireResult{resource: resource, lease: lease, err: err}
+		}()
+	}
+	close(start)
+
+	results := make([]acquireResult, 0, len(resources))
+	leases := make([]*distlock.Lease, 0, len(resources))
+	for range resources {
+		result := requireReceive(t, resultCh, 2*time.Second, "concurrent acquisition did not finish")
+		results = append(results, result)
+		if result.lease != nil {
+			leases = append(leases, result.lease)
+		}
+	}
+	defer func() {
+		for _, lease := range leases {
+			releaseLease(t, lease)
+		}
+	}()
+
+	for _, result := range results {
+		require.NoErrorf(t, result.err, "acquire %q", result.resource)
+		require.NotNilf(t, result.lease, "acquire %q", result.resource)
+		requireContextActive(t, result.lease.Context(), "independent lease was lost")
+	}
+}
+
+func TestLockerExcludesCompetingLeaseUntilRelease(t *testing.T) {
+	cluster := newMemoryCluster(3)
+	firstLocker := newTestLocker(t, cluster, 300*time.Millisecond, retryBackOff)
+	secondLocker := newTestLocker(t, cluster, 300*time.Millisecond, retryBackOff)
+
+	firstLease := acquireLease(t, firstLocker, "resource")
+
+	type acquireResult struct {
+		lease *distlock.Lease
+		err   error
+	}
+	resultCh := make(chan acquireResult, 1)
+	acquireCtx, cancelAcquire := context.WithTimeout(context.Background(), time.Second)
+	defer cancelAcquire()
+	go func() {
+		lease, acquireErr := secondLocker.Acquire(acquireCtx, "resource")
+		resultCh <- acquireResult{lease: lease, err: acquireErr}
+	}()
+
+	requireNoReceive(t, resultCh, 50*time.Millisecond, "competing acquisition returned before release")
+	releaseLease(t, firstLease)
+	require.ErrorIs(t, context.Cause(firstLease.Context()), distlock.ErrLeaseReleased)
+
+	result := requireReceive(t, resultCh, time.Second, "competing acquisition did not finish after release")
+	require.NoError(t, result.err)
+	require.NotNil(t, result.lease)
+	releaseLease(t, result.lease)
+}
+
+func TestLockerContendingAcquireRespectsContext(t *testing.T) {
+	cluster := newMemoryCluster(3)
+	firstLocker := newTestLocker(t, cluster, 300*time.Millisecond, retryBackOff)
+	secondLocker := newTestLocker(t, cluster, 300*time.Millisecond, retryBackOff)
+
+	firstLease := acquireLease(t, firstLocker, "resource")
+	t.Cleanup(func() {
+		if firstLease.Context().Err() != nil {
+			return
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		_ = firstLease.Release(ctx)
+	})
+
+	acquireCtx, cancelAcquire := context.WithTimeout(context.Background(), 75*time.Millisecond)
+	defer cancelAcquire()
+	competingLease, err := secondLocker.Acquire(acquireCtx, "resource")
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+	require.Nil(t, competingLease)
+	requireContextActive(t, firstLease.Context(), "holding lease was affected by a competing acquisition")
+
+	releaseLease(t, firstLease)
+	secondLease := acquireLease(t, secondLocker, "resource")
+	releaseLease(t, secondLease)
+}
+
+func TestLockerAutomaticallyRenewsLease(t *testing.T) {
+	const leaseDuration = 300 * time.Millisecond
+	cluster := newMemoryCluster(3)
+	locker := newTestLocker(t, cluster, leaseDuration, retryBackOff)
+	competingLocker := newTestLocker(t, cluster, leaseDuration, oneAttemptBackOff)
+
+	lease := acquireLease(t, locker, "resource")
+	defer releaseLease(t, lease)
+
+	time.Sleep(leaseDuration + 100*time.Millisecond)
+	requireContextActive(t, lease.Context(), "lease was lost instead of renewed")
+
+	competingLease, err := competingLocker.Acquire(context.Background(), "resource")
+	require.Error(t, err)
+	require.Nil(t, competingLease)
+}
+
+func TestLockerRenewsLeaseWithMinorityUnavailable(t *testing.T) {
+	const leaseDuration = 300 * time.Millisecond
+	cluster := newMemoryCluster(3)
+	locker := newTestLocker(t, cluster, leaseDuration, oneAttemptBackOff)
+	competingLocker := newTestLocker(t, cluster, leaseDuration, oneAttemptBackOff)
+
+	lease := acquireLease(t, locker, "resource")
+	unavailableObserved := cluster.nodes[2].makeUnavailable()
+	originalValidityElapsed := time.NewTimer(leaseDuration + 100*time.Millisecond)
+	defer originalValidityElapsed.Stop()
+	defer func() {
+		cluster.nodes[2].setAvailable(true)
+		releaseLease(t, lease)
+	}()
+
+	requireSignal(t, unavailableObserved, time.Second, "unavailable node did not receive renewal")
+	<-originalValidityElapsed.C
+	requireContextActive(t, lease.Context(), "lease was lost after a minority node became unavailable")
+
+	competingLease, err := competingLocker.Acquire(context.Background(), "resource")
+	require.Error(t, err)
+	require.Nil(t, competingLease)
+}
+
+func TestLockerLosesLeaseWhenNodesRestart(t *testing.T) {
+	const leaseDuration = 300 * time.Millisecond
+	cluster := newMemoryCluster(3)
+	locker := newTestLocker(t, cluster, leaseDuration, oneAttemptBackOff)
+
+	lease := acquireLease(t, locker, "resource")
+	for _, node := range cluster.nodes {
+		node.restart()
+	}
+
+	select {
+	case <-lease.Context().Done():
+		require.ErrorIs(t, context.Cause(lease.Context()), distlock.ErrLeaseLost)
+	case <-time.After(time.Second):
+		require.FailNow(t, "lease was not lost after its node state disappeared")
+	}
+
+	releaseLease(t, lease)
+	require.ErrorIs(t, context.Cause(lease.Context()), distlock.ErrLeaseLost)
+}

pkg/distlock/memory.go

@@ -0,0 +1,139 @@
+package distlock
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"sync"
+	"time"
+)
+
+type memoryLease struct {
+	token     []byte
+	expiresAt time.Time
+}
+
+// MemoryStore stores leases in memory. Its contents are lost when the process exits.
+type MemoryStore struct {
+	mu     sync.Mutex
+	leases map[string]memoryLease
+	now    func() time.Time
+}
+
+// NewMemoryStore creates an empty in-memory lease store.
+func NewMemoryStore() *MemoryStore {
+	return &MemoryStore{
+		leases: make(map[string]memoryLease),
+		now:    time.Now,
+	}
+}
+
+// Acquire creates a lease when the resource does not have an unexpired lease.
+func (s *MemoryStore) Acquire(
+	ctx context.Context, resource string, token []byte, ttl time.Duration,
+) (bool, error) {
+	if err := validateStoreInput(ctx, resource, token, ttl); err != nil {
+		return false, err
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if err := ctx.Err(); err != nil {
+		return false, err
+	}
+	now := s.now()
+
+	if lease, ok := s.leases[resource]; ok && now.Before(lease.expiresAt) {
+		return false, nil
+	}
+
+	s.leases[resource] = memoryLease{
+		token:     bytes.Clone(token),
+		expiresAt: now.Add(ttl),
+	}
+	return true, nil
+}
+
+// Renew extends an unexpired lease when its ownership token matches.
+func (s *MemoryStore) Renew(
+	ctx context.Context, resource string, token []byte, ttl time.Duration,
+) (bool, error) {
+	if err := validateStoreInput(ctx, resource, token, ttl); err != nil {
+		return false, err
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if err := ctx.Err(); err != nil {
+		return false, err
+	}
+	now := s.now()
+
+	lease, ok := s.leases[resource]
+	if !ok {
+		return false, nil
+	}
+	if !now.Before(lease.expiresAt) {
+		delete(s.leases, resource)
+		return false, nil
+	}
+	if !bytes.Equal(lease.token, token) {
+		return false, nil
+	}
+
+	lease.expiresAt = now.Add(ttl)
+	s.leases[resource] = lease
+	return true, nil
+}
+
+// Release removes an unexpired lease when its ownership token matches.
+func (s *MemoryStore) Release(ctx context.Context, resource string, token []byte) (bool, error) {
+	if err := validateStoreResourceToken(ctx, resource, token); err != nil {
+		return false, err
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if err := ctx.Err(); err != nil {
+		return false, err
+	}
+	now := s.now()
+
+	lease, ok := s.leases[resource]
+	if !ok {
+		return false, nil
+	}
+	if !now.Before(lease.expiresAt) {
+		delete(s.leases, resource)
+		return false, nil
+	}
+	if !bytes.Equal(lease.token, token) {
+		return false, nil
+	}
+
+	delete(s.leases, resource)
+	return true, nil
+}
+
+func validateStoreInput(ctx context.Context, resource string, token []byte, ttl time.Duration) error {
+	if err := validateStoreResourceToken(ctx, resource, token); err != nil {
+		return err
+	}
+	if ttl <= 0 {
+		return fmt.Errorf("TTL must be positive")
+	}
+	return nil
+}
+
+func validateStoreResourceToken(ctx context.Context, resource string, token []byte) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if resource == "" {
+		return fmt.Errorf("resource is empty")
+	}
+	if len(token) == 0 {
+		return fmt.Errorf("token is empty")
+	}
+	return nil
+}

pkg/distlock/memory_test.go

@@ -0,0 +1,243 @@
+package distlock
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMemoryStoreAcquire(t *testing.T) {
+	store := newTestMemoryStore()
+	ctx := context.Background()
+
+	acquired, err := store.Acquire(ctx, "resource-a", []byte("owner-a"), time.Minute)
+	require.NoError(t, err)
+	require.True(t, acquired)
+
+	acquired, err = store.Acquire(ctx, "resource-a", []byte("owner-b"), time.Minute)
+	require.NoError(t, err)
+	require.False(t, acquired)
+
+	acquired, err = store.Acquire(ctx, "resource-b", []byte("owner-b"), time.Minute)
+	require.NoError(t, err)
+	require.True(t, acquired)
+}
+
+func TestMemoryStoreExpirationAndStaleOwner(t *testing.T) {
+	now := time.Date(2026, time.August, 26, 12, 0, 0, 0, time.UTC)
+	store := newTestMemoryStore()
+	store.now = func() time.Time { return now }
+	ctx := context.Background()
+	oldToken := []byte("old-owner")
+	newToken := []byte("new-owner")
+
+	acquired, err := store.Acquire(ctx, "resource", oldToken, time.Minute)
+	require.NoError(t, err)
+	require.True(t, acquired)
+
+	// A lease is expired at its expiration time, not only after it.
+	now = now.Add(time.Minute)
+	acquired, err = store.Acquire(ctx, "resource", newToken, time.Minute)
+	require.NoError(t, err)
+	require.True(t, acquired)
+
+	released, err := store.Release(ctx, "resource", oldToken)
+	require.NoError(t, err)
+	require.False(t, released)
+
+	renewed, err := store.Renew(ctx, "resource", oldToken, time.Minute)
+	require.NoError(t, err)
+	require.False(t, renewed)
+
+	released, err = store.Release(ctx, "resource", newToken)
+	require.NoError(t, err)
+	require.True(t, released)
+}
+
+func TestMemoryStoreRenew(t *testing.T) {
+	now := time.Date(2026, time.August, 26, 12, 0, 0, 0, time.UTC)
+	store := newTestMemoryStore()
+	store.now = func() time.Time { return now }
+	ctx := context.Background()
+	token := []byte("owner")
+
+	acquired, err := store.Acquire(ctx, "resource", token, time.Minute)
+	require.NoError(t, err)
+	require.True(t, acquired)
+
+	now = now.Add(30 * time.Second)
+	renewed, err := store.Renew(ctx, "resource", []byte("another-owner"), time.Minute)
+	require.NoError(t, err)
+	require.False(t, renewed)
+
+	renewed, err = store.Renew(ctx, "resource", token, time.Minute)
+	require.NoError(t, err)
+	require.True(t, renewed)
+
+	// The renewal extends the lease from the renewal time.
+	now = now.Add(30 * time.Second)
+	acquired, err = store.Acquire(ctx, "resource", []byte("another-owner"), time.Minute)
+	require.NoError(t, err)
+	require.False(t, acquired)
+
+	now = now.Add(30 * time.Second)
+	renewed, err = store.Renew(ctx, "resource", token, time.Minute)
+	require.NoError(t, err)
+	require.False(t, renewed)
+}
+
+func TestMemoryStoreRelease(t *testing.T) {
+	store := newTestMemoryStore()
+	ctx := context.Background()
+	token := []byte("owner")
+
+	released, err := store.Release(ctx, "resource", token)
+	require.NoError(t, err)
+	require.False(t, released)
+
+	acquired, err := store.Acquire(ctx, "resource", token, time.Minute)
+	require.NoError(t, err)
+	require.True(t, acquired)
+
+	released, err = store.Release(ctx, "resource", []byte("another-owner"))
+	require.NoError(t, err)
+	require.False(t, released)
+
+	released, err = store.Release(ctx, "resource", token)
+	require.NoError(t, err)
+	require.True(t, released)
+
+	released, err = store.Release(ctx, "resource", token)
+	require.NoError(t, err)
+	require.False(t, released)
+}
+
+func TestMemoryStoreCopiesToken(t *testing.T) {
+	store := newTestMemoryStore()
+	ctx := context.Background()
+	token := []byte("owner")
+	originalToken := append([]byte(nil), token...)
+
+	acquired, err := store.Acquire(ctx, "resource", token, time.Minute)
+	require.NoError(t, err)
+	require.True(t, acquired)
+
+	token[0] = 'x'
+	released, err := store.Release(ctx, "resource", originalToken)
+	require.NoError(t, err)
+	require.True(t, released)
+}
+
+func TestMemoryStoreValidation(t *testing.T) {
+	tests := []struct {
+		name string
+		run  func(*MemoryStore) error
+	}{
+		{
+			name: "acquire with empty resource",
+			run: func(store *MemoryStore) error {
+				_, err := store.Acquire(context.Background(), "", []byte("owner"), time.Minute)
+				return err
+			},
+		},
+		{
+			name: "acquire with empty token",
+			run: func(store *MemoryStore) error {
+				_, err := store.Acquire(context.Background(), "resource", nil, time.Minute)
+				return err
+			},
+		},
+		{
+			name: "acquire with zero TTL",
+			run: func(store *MemoryStore) error {
+				_, err := store.Acquire(context.Background(), "resource", []byte("owner"), 0)
+				return err
+			},
+		},
+		{
+			name: "renew with negative TTL",
+			run: func(store *MemoryStore) error {
+				_, err := store.Renew(context.Background(), "resource", []byte("owner"), -time.Second)
+				return err
+			},
+		},
+		{
+			name: "release with empty token",
+			run: func(store *MemoryStore) error {
+				_, err := store.Release(context.Background(), "resource", nil)
+				return err
+			},
+		},
+		{
+			name: "cancelled context",
+			run: func(store *MemoryStore) error {
+				ctx, cancel := context.WithCancel(context.Background())
+				cancel()
+				_, err := store.Acquire(ctx, "resource", []byte("owner"), time.Minute)
+				return err
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			store := newTestMemoryStore()
+			require.Error(t, test.run(store))
+
+			// Invalid operations must not create or replace a lease.
+			acquired, err := store.Acquire(context.Background(), "resource", []byte("valid-owner"), time.Minute)
+			require.NoError(t, err)
+			require.True(t, acquired)
+		})
+	}
+}
+
+func TestMemoryStoreConcurrentAcquire(t *testing.T) {
+	const attempts = 100
+
+	store := newTestMemoryStore()
+	ctx := context.Background()
+	start := make(chan struct{})
+	errCh := make(chan error, attempts)
+	var acquired atomic.Int64
+	var wg sync.WaitGroup
+
+	for i := range attempts {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			<-start
+
+			ok, err := store.Acquire(ctx, "resource", []byte(fmt.Sprintf("owner-%d", i)), time.Minute)
+			if err != nil {
+				errCh <- err
+				return
+			}
+			if ok {
+				acquired.Add(1)
+			}
+		}()
+	}
+
+	close(start)
+	wg.Wait()
+	close(errCh)
+
+	for err := range errCh {
+		require.NoError(t, err)
+	}
+	require.EqualValues(t, 1, acquired.Load())
+}
+
+func newTestMemoryStore() *MemoryStore {
+	store := NewMemoryStore()
+	store.now = func() time.Time {
+		return time.Date(2026, time.August, 26, 12, 0, 0, 0, time.UTC)
+	}
+	return store
+}

pkg/distlock/node.go

@@ -0,0 +1,29 @@
+package distlock
+
+import (
+	"context"
+	"time"
+)
+
+// Cluster provides point-in-time snapshots of the independent nodes that participate in distributed leases.
+// Implementations must be safe for concurrent use.
+type Cluster interface {
+	// Nodes returns each node counted toward quorum exactly once, including nodes that are temporarily unavailable.
+	// The Locker may keep and use the returned nodes throughout acquisition and until any acquired lease is released.
+	Nodes(ctx context.Context) ([]Node, error)
+}
+
+// Node provides lease operations on a single cluster node.
+//
+// Each operation performs one attempt. A true result means the operation took effect. A false result with no error
+// means the node responded but its lease state rejected the operation. If err is non-nil, the result is unknown and
+// the boolean result must be ignored. Implementations must return promptly when ctx is cancelled, not modify token,
+// and be safe for concurrent use.
+type Node interface {
+	// Acquire creates a lease when resource does not have an unexpired lease.
+	Acquire(ctx context.Context, resource string, token []byte, ttl time.Duration) (bool, error)
+	// Renew extends an unexpired lease when its ownership token matches.
+	Renew(ctx context.Context, resource string, token []byte, ttl time.Duration) (bool, error)
+	// Release removes an unexpired lease when its ownership token matches.
+	Release(ctx context.Context, resource string, token []byte) (bool, error)
+}

pkg/distlock/store.go

@@ -0,0 +1,16 @@
+package distlock
+
+import (
+	"context"
+	"time"
+)
+
+// Store holds node-local lease state and provides atomic operations over it.
+type Store interface {
+	// Acquire creates a lease when the resource does not have an unexpired lease.
+	Acquire(ctx context.Context, resource string, token []byte, ttl time.Duration) (bool, error)
+	// Renew extends an unexpired lease when its ownership token matches.
+	Renew(ctx context.Context, resource string, token []byte, ttl time.Duration) (bool, error)
+	// Release removes a lease when its ownership token matches.
+	Release(ctx context.Context, resource string, token []byte) (bool, error)
+}

website/docs/2-getting-started/1-install-cli.md

@@ -57,6 +57,12 @@ curl -fsS https://get.uncloud.run/install.sh | VERSION=nightly sh
 
 More information about nightly builds of the CLI and daemon can be found [here](https://github.com/psviderski/uncloud/releases/tag/nightly).
 
+## mise
+You can install uncloud with [mise](https://mise.jdx.dev/):
+```
+mise use github:psviderski/uncloud[exe=uc]
+```
+
 ## GitHub download (macOS, Linux)
 
 You can manually download and use a pre-built binary from the

website/docs/8-compose-file-reference/1-support-matrix.md

@@ -45,9 +45,11 @@ If you rely on a specific Compose feature that is not supported by Uncloud, plea
 | `secrets`                        | ⚠️ Limited         | Reference secrets in `environment`, see [Secrets](../3-concepts/8-secrets.md). File mounts not supported                                   |
 | `security_opt`                   | ❌ Not supported    |                                                                                                                                            |
 | `shm_size`                       | ✅ Supported        | Shared memory size                                                                                                                         |
+| `stdin_open`                     | ✅ Supported        | Allocate standard input and keep it open                                                                                                   |
 | `stop_grace_period`              | ✅ Supported        | Time to wait after SIGTERM before SIGKILL                                                                                                  |
 | `storage_opt`                    | ❌ Not supported    |                                                                                                                                            |
 | `sysctls`                        | ✅ Supported        | Namespaced kernel parameters                                                                                                               |
+| `tty`                            | ✅ Supported        | Allocate a pseudo-TTY and connect the container's standard streams to it                                                                   |
 | `ulimits`                        | ✅ Supported        | Resource limits                                                                                                                            |
 | `user`                           | ✅ Supported        | Set container user                                                                                                                         |
 | `volumes`                        | ✅ Supported        | Named volumes, bind mounts, tmpfs                                                                                                          |