chore(.editorconfig): ignore formatting for release notes

Signed-off-by: Miek Gieben <miek@miek.nl>

.editorconfig

@@ -18,3 +18,6 @@ indent_size = 4
 [*.sh]
 indent_style = space
 indent_size = 4
+
+[release-notes-*.md]
+max_line_length = off

README.md

@@ -44,6 +44,8 @@ complexity of Kubernetes.
   [Uncloud DNS](https://github.com/psviderski/uncloud-dns) service.
 * **Automatic HTTPS**: Built-in Caddy reverse proxy handles TLS certificate provisioning and renewal using Let's
   Encrypt.
+* **Cluster storage for TLS certificates**: Native cluster storage for Caddy shares certificates and ACME challenge
+  tokens across machines and coordinates issuance.
 * **Docker-like CLI**: Familiar commands for managing both infrastructure and applications.
 * **Remote management**: Control your entire infrastructure through SSH access to any single machine in the cluster.
 

api/pb/machine.proto

@@ -20,7 +20,7 @@ service Machine {
   // InspectMachine retrieves detailed information about the machine. Supports broadcasting to multiple machines.
   rpc InspectMachine(google.protobuf.Empty) returns (InspectMachineResponse);
   // WaitForStoreVersion waits until the cluster store on this machine has reached each requested actor version, with
-  // no known missing or pending transactions through those versions.
+  // no known missing or pending transactions through those versions from active members.
   // Corrosion may satisfy a version by applying its surviving changes or by marking it complete because its changes
   // have been superseded.
   //

api/pb/machine_grpc.pb.go

@@ -48,7 +48,7 @@ type MachineClient interface {
 	// InspectMachine retrieves detailed information about the machine. Supports broadcasting to multiple machines.
 	InspectMachine(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*InspectMachineResponse, error)
 	// WaitForStoreVersion waits until the cluster store on this machine has reached each requested actor version, with
-	// no known missing or pending transactions through those versions.
+	// no known missing or pending transactions through those versions from active members.
 	// Corrosion may satisfy a version by applying its surviving changes or by marking it complete because its changes
 	// have been superseded.
 	//
@@ -224,7 +224,7 @@ type MachineServer interface {
 	// InspectMachine retrieves detailed information about the machine. Supports broadcasting to multiple machines.
 	InspectMachine(context.Context, *emptypb.Empty) (*InspectMachineResponse, error)
 	// WaitForStoreVersion waits until the cluster store on this machine has reached each requested actor version, with
-	// no known missing or pending transactions through those versions.
+	// no known missing or pending transactions through those versions from active members.
 	// Corrosion may satisfy a version by applying its surviving changes or by marking it complete because its changes
 	// have been superseded.
 	//

cmd/uc/caddy/cert/ls.go

@@ -0,0 +1,167 @@
+package cert
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"charm.land/lipgloss/v2"
+	"github.com/caddyserver/certmagic"
+	"github.com/docker/go-units"
+	"github.com/psviderski/uncloud/internal/cli"
+	"github.com/psviderski/uncloud/internal/cli/completion"
+	"github.com/psviderski/uncloud/internal/cli/tui"
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/psviderski/uncloud/pkg/client"
+	"github.com/spf13/cobra"
+)
+
+type listOptions struct {
+	machine string
+	output  string
+}
+
+func NewListCommand() *cobra.Command {
+	opts := listOptions{}
+	cmd := &cobra.Command{
+		Use:     "ls",
+		Aliases: []string{"list"},
+		Short:   "List certificates in cluster storage for Caddy.",
+		Long: `List certificates stored in the Uncloud cluster storage for Caddy.
+
+Caddy must use the Uncloud storage module (https://github.com/unlabs-dev/caddy-uncloud)
+configured with 'storage uncloud' in the global options for its certificates to appear here.
+
+This inventory does not check whether Caddy currently serves or trusts a certificate.`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			uncli := cmd.Context().Value("cli").(*cli.CLI)
+			return list(cmd.Context(), uncli, opts)
+		},
+	}
+
+	cmd.Flags().StringVarP(&opts.machine, "machine", "m", "",
+		"Name or ID of the machine to read certificate storage from. (default is connected machine)")
+	cmd.Flags().StringVarP(&opts.output, "output", "o", "",
+		"Output format: 'json' or empty for a human-readable table.")
+	completion.MachinesFlag(cmd)
+
+	return cmd
+}
+
+func list(ctx context.Context, uncli *cli.CLI, opts listOptions) error {
+	if opts.output != "" && opts.output != "json" {
+		return fmt.Errorf("unsupported output format '%s' (supported: json)", opts.output)
+	}
+
+	clusterClient, err := uncli.ConnectCluster(ctx)
+	if err != nil {
+		return fmt.Errorf("connect to cluster: %w", err)
+	}
+	defer clusterClient.Close()
+
+	certs, listErr := clusterClient.Caddy.ListCertificates(ctx,
+		client.CaddyListCertificatesOptions{Machine: opts.machine})
+	if len(certs) > 0 || listErr == nil {
+		if err = printCertificates(os.Stdout, certs, opts.output); err != nil {
+			return fmt.Errorf("print Caddy certificates: %w", err)
+		}
+	}
+	if listErr != nil {
+		if len(certs) > 0 {
+			return fmt.Errorf("certificate list is partial: %w", listErr)
+		}
+		return fmt.Errorf("list Caddy certificates: %w", listErr)
+	}
+
+	return nil
+}
+
+func printCertificates(out io.Writer, certs []api.IssuedCertificate, output string) error {
+	type certificate struct {
+		api.IssuedCertificate
+		Fingerprint string
+	}
+
+	items := make([]certificate, 0, len(certs))
+	for _, cert := range certs {
+		fingerprint := cert.Fingerprint()
+		items = append(items, certificate{
+			IssuedCertificate: cert,
+			Fingerprint:       hex.EncodeToString(fingerprint[:]),
+		})
+	}
+	slices.SortFunc(items, func(a, b certificate) int {
+		if cmp := strings.Compare(a.SAN, b.SAN); cmp != 0 {
+			return cmp
+		}
+		if cmp := a.Chain[0].NotAfter.Compare(b.Chain[0].NotAfter); cmp != 0 {
+			return cmp
+		}
+		return strings.Compare(a.Fingerprint, b.Fingerprint)
+	})
+
+	if output == "json" {
+		encoder := json.NewEncoder(out)
+		encoder.SetIndent("", "  ")
+		if err := encoder.Encode(items); err != nil {
+			return fmt.Errorf("marshal certificates: %w", err)
+		}
+		return nil
+	}
+
+	if len(items) == 0 {
+		_, err := fmt.Fprintln(out, "No Caddy certificates found in cluster storage.")
+		return err
+	}
+
+	t := tui.NewTable()
+	t.Headers("ID", "NAME", "ISSUER", "EXPIRES")
+	for _, item := range items {
+		t.Row(
+			item.Fingerprint[:12],
+			item.SAN,
+			formatIssuer(item.IssuerData),
+			formatExpiry(item.Chain[0].NotAfter),
+		)
+	}
+	_, err := lipgloss.Fprintln(out, t)
+	return err
+}
+
+func formatIssuer(data api.CertificateIssuerData) string {
+	if data.ACME == nil {
+		return "unknown"
+	}
+
+	switch data.ACME.CA {
+	case certmagic.LetsEncryptProductionCA:
+		return "Let's Encrypt"
+	case certmagic.LetsEncryptStagingCA:
+		return "Let's Encrypt (staging)"
+	case certmagic.ZeroSSLProductionCA:
+		return "ZeroSSL"
+	case certmagic.GoogleTrustProductionCA:
+		return "Google Trust Services"
+	case certmagic.GoogleTrustStagingCA:
+		return "Google Trust Services (staging)"
+	default:
+		return data.ACME.CA
+	}
+}
+
+func formatExpiry(expires time.Time) string {
+	delta := expires.Sub(time.Now())
+	if delta <= 0 {
+		return fmt.Sprintf("%s (expired %s ago)", expires.UTC().Format(time.DateOnly),
+			strings.ToLower(units.HumanDuration(-delta)))
+	}
+	return fmt.Sprintf("%s (%s)", expires.UTC().Format(time.DateOnly),
+		strings.ToLower(units.HumanDuration(delta)))
+}

cmd/uc/caddy/cert/ls_test.go

@@ -0,0 +1,128 @@
+package cert
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/caddyserver/certmagic"
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPrintCertificates(t *testing.T) {
+	now := time.Now().UTC().Truncate(time.Hour)
+	certs := []api.IssuedCertificate{
+		testIssuedCertificate("z.example.com", "z", now.Add(48*time.Hour), certmagic.LetsEncryptProductionCA),
+		testIssuedCertificate("a.example.com", "old", now.Add(-48*time.Hour), certmagic.LetsEncryptStagingCA),
+		testIssuedCertificate("a.example.com", "new", now.Add(72*time.Hour), "https://ca.example/directory"),
+		testIssuedCertificate("unknown.example.com", "unknown", now.Add(96*time.Hour), ""),
+	}
+
+	var table bytes.Buffer
+	require.NoError(t, printCertificates(&table, certs, ""))
+	output := table.String()
+	assert.Contains(t, output, "ID")
+	assert.Contains(t, output, "NAME")
+	assert.Contains(t, output, "ISSUER")
+	assert.Contains(t, output, "EXPIRES")
+	assert.Contains(t, output, now.Add(-48*time.Hour).Format(time.DateOnly)+" (expired ")
+	assert.Contains(t, output, now.Add(48*time.Hour).Format(time.DateOnly)+" (")
+	assert.Contains(t, output, "Let's Encrypt (staging)")
+	assert.Contains(t, output, "https://ca.example/directory")
+	assert.Less(t, strings.Index(output, now.Add(-48*time.Hour).Format(time.DateOnly)),
+		strings.Index(output, now.Add(72*time.Hour).Format(time.DateOnly)))
+	assert.Less(t, strings.Index(output, "a.example.com"), strings.Index(output, "z.example.com"))
+	assert.Contains(t, output, testFingerprint("old")[:12])
+
+	var encoded bytes.Buffer
+	require.NoError(t, printCertificates(&encoded, certs, "json"))
+	var items []struct {
+		api.IssuedCertificate
+		Fingerprint string
+	}
+	require.NoError(t, json.Unmarshal(encoded.Bytes(), &items))
+	require.Len(t, items, 4)
+	assert.Equal(t, testFingerprint("old"), items[0].Fingerprint)
+	assert.Equal(t, "a.example.com", items[0].SAN)
+	require.Len(t, items[0].Chain, 1)
+	assert.Equal(t, []byte("old"), items[0].Chain[0].Raw)
+	assert.Equal(t, now.Add(-48*time.Hour), items[0].Chain[0].NotAfter)
+	require.NotNil(t, items[0].IssuerData.ACME)
+	assert.Equal(t, certmagic.LetsEncryptStagingCA, items[0].IssuerData.ACME.CA)
+	assert.Equal(t, testFingerprint("new"), items[1].Fingerprint)
+	require.NotNil(t, items[1].IssuerData.ACME)
+	assert.Equal(t, "https://ca.example/directory", items[1].IssuerData.ACME.CA)
+	assert.Nil(t, items[2].IssuerData.ACME)
+	assert.Equal(t, "z.example.com", items[3].SAN)
+}
+
+func TestPrintCertificates_Empty(t *testing.T) {
+	for _, tt := range []struct {
+		output string
+		want   string
+	}{
+		{output: "", want: "No Caddy certificates found in cluster storage.\n"},
+		{output: "json", want: "[]\n"},
+	} {
+		var out bytes.Buffer
+		require.NoError(t, printCertificates(&out, nil, tt.output))
+		assert.Equal(t, tt.want, out.String())
+	}
+}
+
+func TestFormatIssuer(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		ca   string
+		want string
+	}{
+		{name: "production", ca: certmagic.LetsEncryptProductionCA, want: "Let's Encrypt"},
+		{name: "staging", ca: certmagic.LetsEncryptStagingCA, want: "Let's Encrypt (staging)"},
+		{name: "ZeroSSL", ca: certmagic.ZeroSSLProductionCA, want: "ZeroSSL"},
+		{name: "Google Trust Services", ca: certmagic.GoogleTrustProductionCA, want: "Google Trust Services"},
+		{name: "Google Trust Services staging", ca: certmagic.GoogleTrustStagingCA, want: "Google Trust Services (staging)"},
+		{name: "other", ca: "https://ca.example/directory", want: "https://ca.example/directory"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, formatIssuer(api.CertificateIssuerData{
+				ACME: &api.ACMEIssuerData{CA: tt.ca},
+			}))
+		})
+	}
+
+	assert.Equal(t, "unknown", formatIssuer(api.CertificateIssuerData{}))
+}
+
+func TestFormatExpiry_UTC(t *testing.T) {
+	now := time.Now().UTC()
+	brisbane := time.FixedZone("AEST", 10*60*60)
+	expires := time.Date(now.Year(), now.Month(), now.Day()+3, 3, 0, 0, 0, brisbane)
+	assert.True(t, strings.HasPrefix(formatExpiry(expires), expires.UTC().Format(time.DateOnly)+" ("))
+	assert.NotEqual(t, expires.Format(time.DateOnly), expires.UTC().Format(time.DateOnly))
+	expired := now.Add(-2 * time.Hour)
+	assert.True(t, strings.HasPrefix(formatExpiry(expired), expired.UTC().Format(time.DateOnly)+" (expired "))
+	assert.Contains(t, formatExpiry(expired), " ago)")
+}
+
+func testIssuedCertificate(name, raw string, expires time.Time, ca string) api.IssuedCertificate {
+	cert := api.IssuedCertificate{
+		SAN:   name,
+		Chain: []*x509.Certificate{{Raw: []byte(raw), NotAfter: expires}},
+	}
+	if ca != "" {
+		cert.IssuerData.ACME = &api.ACMEIssuerData{CA: ca}
+	}
+	return cert
+}
+
+func testFingerprint(raw string) string {
+	sum := sha256.Sum256([]byte(raw))
+	return hex.EncodeToString(sum[:])
+}

cmd/uc/caddy/cert/root.go

@@ -0,0 +1,12 @@
+package cert
+
+import "github.com/spf13/cobra"
+
+func NewRootCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "cert",
+		Short: "Inspect certificates in cluster storage for Caddy.",
+	}
+	cmd.AddCommand(NewListCommand())
+	return cmd
+}

cmd/uc/caddy/root.go

@@ -1,6 +1,7 @@
 package caddy
 
 import (
+	"github.com/psviderski/uncloud/cmd/uc/caddy/cert"
 	"github.com/spf13/cobra"
 )
 
@@ -10,6 +11,7 @@ func NewRootCommand() *cobra.Command {
 		Short: "Manage Caddy reverse proxy service.",
 	}
 	cmd.AddCommand(
+		cert.NewRootCommand(),
 		NewConfigCommand(),
 		NewDeployCommand(),
 		NewLogsCommand(),

cmd/uc/dns/set.go

@@ -9,8 +9,10 @@ import (
 	"github.com/psviderski/uncloud/api/pb"
 	"github.com/psviderski/uncloud/internal/cli"
 	"github.com/psviderski/uncloud/internal/cli/tui"
+	"github.com/psviderski/uncloud/internal/grpcversion"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 )
 
@@ -49,6 +51,7 @@ func set(ctx context.Context, uncli *cli.CLI, name string) error {
 	}
 	defer clusterClient.Close()
 
+	ctx = metadata.AppendToOutgoingContext(ctx, grpcversion.MetadataKeyMinServerVersion, "0.21.0")
 	_, err = clusterClient.SetDomain(ctx, &pb.SetDomainRequest{Name: name})
 	if err != nil {
 		if status.Convert(err).Code() == codes.AlreadyExists {

internal/corrosion/admin.go

@@ -13,6 +13,8 @@ import (
 	"time"
 )
 
+const adminCommandTimeout = 5 * time.Second
+
 // AdminClient is a client for the Corrosion admin API.
 type AdminClient struct {
 	sockPath string
@@ -32,10 +34,15 @@ type Response struct {
 // The channel will be closed after sending the last or error response. The caller must read from the channel until
 // it is closed.
 func (c *AdminClient) SendCommand(cmd []byte) (<-chan Response, error) {
-	conn, err := net.Dial("unix", c.sockPath)
+	// TODO: Accept contexts in admin methods so callers can cancel requests and set their deadlines.
+	conn, err := net.DialTimeout("unix", c.sockPath, adminCommandTimeout)
 	if err != nil {
 		return nil, fmt.Errorf("connect to admin socket: %w", err)
 	}
+	if err = conn.SetDeadline(time.Now().Add(adminCommandTimeout)); err != nil {
+		conn.Close()
+		return nil, fmt.Errorf("set admin connection deadline: %w", err)
+	}
 
 	if _, err = conn.Write(encodeFrame(cmd)); err != nil {
 		conn.Close()

internal/machine/machine.go

@@ -262,11 +262,11 @@ func NewMachine(config *Config) (*Machine, error) {
 	if err != nil {
 		return nil, fmt.Errorf("create corrosion API client: %w", err)
 	}
-	corroStore := store.New(corro)
 	corroAdmin, err := corrosion.NewAdminClient(config.CorrosionAdminSockPath)
 	if err != nil {
 		return nil, fmt.Errorf("create corrosion admin client: %w", err)
 	}
+	corroStore := store.New(corro, corroAdmin)
 
 	initialised := make(chan struct{})
 	clusterReady := make(chan struct{})

internal/machine/store/replication.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/psviderski/uncloud/internal/corrosion"
 	"github.com/psviderski/uncloud/pkg/api"
 )
 
@@ -15,8 +16,8 @@ import (
 var ErrInvalidStoreVersion = errors.New("invalid store version")
 
 // WaitForVersion waits until the local store has reached each actor's minimum version in minVersion, with no known
-// missing or pending transactions through those versions. Corrosion may satisfy a version by applying its surviving
-// changes or by marking it complete because its changes have been superseded.
+// missing or pending transactions through those versions from active members. Corrosion may satisfy a version
+// by applying its surviving changes or by marking it complete because its changes have been superseded.
 //
 // Waiting normally makes the captured data available locally. However, if another write replaces some of that data
 // before it arrives, Corrosion can complete the older version without transferring the replaced data. If the
@@ -53,28 +54,64 @@ func (s *Store) WaitForVersion(ctx context.Context, minVersion api.StoreVersion)
 		return nil
 	}
 
-	// Check all bookkeeping in one SQLite snapshot. Raw buffered rows can remain after application,
-	// but sequence bookkeeping is removed in the same transaction that applies or clears a version.
-	query := `WITH min_version(actor_id, version) AS (VALUES ` + strings.Join(placeholders, ", ") + `)
+	queryPrefix := `WITH min_version(actor_id, version) AS (VALUES ` + strings.Join(placeholders, ", ") + `)
 		SELECT NOT EXISTS (
 			SELECT 1 FROM min_version
 			LEFT JOIN crsql_db_versions AS current ON current.site_id = min_version.actor_id
-			WHERE COALESCE(current.db_version, 0) < min_version.version
-				OR EXISTS (
-					SELECT 1 FROM __corro_bookkeeping_gaps
-					WHERE actor_id = min_version.actor_id AND start <= min_version.version
-				)
-				OR EXISTS (
-					SELECT 1 FROM __corro_seq_bookkeeping
-					WHERE site_id = min_version.actor_id AND db_version <= min_version.version
-				)
-		)`
+			WHERE COALESCE(current.db_version, 0) < min_version.version`
+
+	versionsReached := false
 
 	ticker := time.NewTicker(100 * time.Millisecond)
 	defer ticker.Stop()
 
 	for {
-		rows, err := s.corro.QueryContext(ctx, query, args...)
+		query := queryPrefix
+		queryArgs := args
+		// Check for gaps and pending transactions only after the requested versions are reached.
+		if versionsReached {
+			states, err := s.corroAdmin.ClusterMembershipStates(true)
+			if err != nil {
+				return fmt.Errorf("get cluster membership for store replication: %w", err)
+			}
+
+			activePlaceholders := make([]string, 0, len(states))
+			queryArgs = make([]any, len(args), len(args)+len(states))
+			copy(queryArgs, args)
+			for _, state := range states {
+				if state.State != corrosion.MembershipStateAlive && state.State != corrosion.MembershipStateSuspect {
+					continue
+				}
+				actorID, err := uuid.Parse(state.ID)
+				if err != nil {
+					return fmt.Errorf("parse cluster member actor '%s': %w", state.ID, err)
+				}
+				activePlaceholders = append(activePlaceholders, "?")
+				queryArgs = append(queryArgs, [16]byte(actorID))
+			}
+			if len(activePlaceholders) > 0 {
+				// Check active members' gaps and pending sequences in the same query for simplicity.
+				// Buffered rows can remain in __corro_buffered_changes after application, but sequence bookkeeping
+				// is removed when a version completes.
+				query += `
+				OR (
+					min_version.actor_id IN (` + strings.Join(activePlaceholders, ", ") + `)
+					AND (
+						EXISTS (
+							SELECT 1 FROM __corro_bookkeeping_gaps
+							WHERE actor_id = min_version.actor_id AND start <= min_version.version
+						)
+						OR EXISTS (
+							SELECT 1 FROM __corro_seq_bookkeeping
+							WHERE site_id = min_version.actor_id AND db_version <= min_version.version
+						)
+					)
+				)`
+			}
+		}
+		query += ")"
+
+		rows, err := s.corro.QueryContext(ctx, query, queryArgs...)
 		if err != nil {
 			return fmt.Errorf("check store replication: %w", err)
 		}
@@ -84,6 +121,7 @@ func (s *Store) WaitForVersion(ctx context.Context, minVersion api.StoreVersion)
 			}
 			return errors.New("check store replication: store version query returned no rows")
 		}
+
 		var reached int
 		if err = rows.Scan(&reached); err != nil {
 			rows.Close()
@@ -95,7 +133,11 @@ func (s *Store) WaitForVersion(ctx context.Context, minVersion api.StoreVersion)
 			return fmt.Errorf("check store replication: %w", err)
 		}
 		if reached == 1 {
-			return ctx.Err()
+			if versionsReached {
+				return nil
+			}
+			versionsReached = true
+			continue
 		}
 
 		select {

internal/machine/store/store.go

@@ -24,11 +24,12 @@ var (
 
 // Store is a cluster store backed by a distributed Corrosion database.
 type Store struct {
-	corro *corrosion.APIClient
+	corro      *corrosion.APIClient
+	corroAdmin *corrosion.AdminClient
 }
 
-func New(corro *corrosion.APIClient) *Store {
-	return &Store{corro: corro}
+func New(corro *corrosion.APIClient, corroAdmin *corrosion.AdminClient) *Store {
+	return &Store{corro: corro, corroAdmin: corroAdmin}
 }
 
 // Get retrieves an unnamespaced legacy value.

pkg/api/caddy_certificate.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"crypto/sha256"
 	"crypto/x509"
 	"encoding/json"
 	"encoding/pem"
@@ -23,6 +24,11 @@ type IssuedCertificate struct {
 	IssuerData CertificateIssuerData
 }
 
+// Fingerprint returns the SHA-256 fingerprint of the leaf certificate.
+func (c IssuedCertificate) Fingerprint() [sha256.Size]byte {
+	return sha256.Sum256(c.Chain[0].Raw)
+}
+
 // CertificateIssuerData preserves issuer-specific metadata and provides a best-effort typed view of ACME records.
 type CertificateIssuerData struct {
 	// Raw is the original issuer_data JSON, including unrecognized formats and fields.

pkg/api/caddy_certificate_test.go

@@ -4,6 +4,7 @@ import (
 	"crypto/ed25519"
 	"crypto/rand"
 	"crypto/x509"
+	"encoding/hex"
 	"encoding/pem"
 	"math/big"
 	"testing"
@@ -45,6 +46,17 @@ func TestIssuedCertificateFromProto(t *testing.T) {
 	}
 }
 
+func TestIssuedCertificateFingerprint(t *testing.T) {
+	cert := IssuedCertificate{Chain: []*x509.Certificate{
+		{Raw: []byte("abc")},
+		{Raw: []byte("issuer")},
+	}}
+	want, err := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
+	require.NoError(t, err)
+	fingerprint := cert.Fingerprint()
+	assert.Equal(t, want, fingerprint[:])
+}
+
 func TestIssuedCertificateFromProto_InvalidCertificate(t *testing.T) {
 	valid := testCertificatePEM(t, 1)
 	invalidDER := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte("invalid")})
@@ -57,15 +69,25 @@ func TestIssuedCertificateFromProto_InvalidCertificate(t *testing.T) {
 		{name: "missing SAN", input: &pb.IssuedCertificate{Chain: valid}, wantErr: "missing SAN"},
 		{name: "blank SAN", input: &pb.IssuedCertificate{San: " \t", Chain: valid}, wantErr: "missing SAN"},
 		{name: "empty chain", input: &pb.IssuedCertificate{San: "app.example.com"}, wantErr: "empty certificate chain"},
-		{name: "invalid PEM", input: &pb.IssuedCertificate{San: "app.example.com", Chain: []byte("not PEM")},
-			wantErr: "empty certificate chain"},
-		{name: "wrong PEM block type", input: &pb.IssuedCertificate{San: "app.example.com",
-			Chain: pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: []byte("key")})},
-			wantErr: "unexpected PEM block type"},
-		{name: "invalid DER", input: &pb.IssuedCertificate{San: "app.example.com", Chain: invalidDER},
-			wantErr: "parse X.509 certificate"},
-		{name: "invalid certificate after valid leaf", input: &pb.IssuedCertificate{San: "app.example.com",
-			Chain: append(append([]byte(nil), valid...), invalidDER...)}, wantErr: "parse X.509 certificate"},
+		{
+			name: "invalid PEM", input: &pb.IssuedCertificate{San: "app.example.com", Chain: []byte("not PEM")},
+			wantErr: "empty certificate chain",
+		},
+		{
+			name: "wrong PEM block type", input: &pb.IssuedCertificate{
+				San:   "app.example.com",
+				Chain: pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: []byte("key")}),
+			},
+			wantErr: "unexpected PEM block type",
+		},
+		{
+			name: "invalid DER", input: &pb.IssuedCertificate{San: "app.example.com", Chain: invalidDER},
+			wantErr: "parse X.509 certificate",
+		},
+		{name: "invalid certificate after valid leaf", input: &pb.IssuedCertificate{
+			San:   "app.example.com",
+			Chain: append(append([]byte(nil), valid...), invalidDER...),
+		}, wantErr: "parse X.509 certificate"},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			cert, err := IssuedCertificateFromProto(tt.input)
@@ -126,8 +148,10 @@ func TestIssuedCertificateFromProto_IssuerData(t *testing.T) {
 		{name: "missing certificate URL", raw: `{"ca":"https://ca.example/directory"}`},
 		{name: "invalid JSON", raw: "{"},
 		{name: "invalid field types", raw: `{"url":123,"ca":false}`},
-		{name: "malformed renewal information",
-			raw: `{"url":"https://ca.example/cert/1","ca":"https://ca.example/directory","renewal_info":{"_selectedTime":"invalid"}}`},
+		{
+			name: "malformed renewal information",
+			raw:  `{"url":"https://ca.example/cert/1","ca":"https://ca.example/directory","renewal_info":{"_selectedTime":"invalid"}}`,
+		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			cert, err := IssuedCertificateFromProto(&pb.IssuedCertificate{

pkg/client/caddy.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"regexp"
+	"time"
 
 	"github.com/Masterminds/semver"
 	"github.com/distribution/reference"
@@ -66,6 +67,7 @@ func (c *CaddyClient) Config(ctx context.Context, opts CaddyConfigOptions) (api.
 // CaddyDeploymentOptions configures a Caddy reverse proxy deployment.
 type CaddyDeploymentOptions struct {
 	// Image defaults to the latest stable 2.x.x official Caddy image.
+	// Custom images must include curl and start Caddy with /etc/caddy/Caddyfile.
 	Image string
 	// Config contains an optional global Caddyfile.
 	Config    string
@@ -91,17 +93,32 @@ func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOpt
 
 	spec := api.ServiceSpec{
 		Container: api.ContainerSpec{
-			Command: []string{"caddy", "run", "-c", "/config/Caddyfile"},
 			Env: map[string]string{
 				"CADDY_ADMIN": "unix//run/caddy/admin.sock",
 			},
+			Healthcheck: &api.HealthcheckSpec{
+				Test: []string{
+					"CMD",
+					"curl", "-fsS",
+					"-o", "/dev/null",
+					"--unix-socket", "/run/caddy/admin.sock",
+					"http://localhost/config/",
+				},
+				Interval:      30 * time.Second,
+				Timeout:       5 * time.Second,
+				Retries:       3,
+				StartPeriod:   10 * time.Second,
+				StartInterval: 1 * time.Second,
+			},
 			Image: image,
 			VolumeMounts: []api.VolumeMount{
 				{
 					VolumeName:    "data",
-					ContainerPath: "/config",
+					ContainerPath: "/etc/caddy",
+					ReadOnly:      true,
 				},
 				{
+					// Keep local TLS assets persistent while Caddy uses the default file system storage.
 					VolumeName:    "data",
 					ContainerPath: "/data",
 				},

pkg/client/machine.go

@@ -143,7 +143,7 @@ func (cli *Client) WaitClusterReady(ctx context.Context, timeout time.Duration)
 }
 
 // WaitForStoreVersion waits until the cluster store on the target machine has reached each requested actor version
-// in minVersion, with no known missing or pending transactions through those versions.
+// in minVersion, with no known missing or pending transactions through those versions from active members.
 // The context controls cancellation and the deadline. An empty minVersion requires no replication.
 // This method observes replication without initiating synchronisation.
 //

pkg/client/service.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"maps"
 	"slices"
 	"sync"
 
@@ -320,40 +321,18 @@ func (cli *Client) StartService(ctx context.Context, id string) error {
 
 // ListServices returns a list of all services and their containers.
 func (cli *Client) ListServices(ctx context.Context) ([]api.Service, error) {
-	machines, err := cli.ListMachines(ctx, nil)
-	if err != nil {
-		return nil, fmt.Errorf("list machines: %w", err)
-	}
-
-	// Broadcast the container list request to all available machines.
-	md := metadata.New(nil)
-	for _, m := range machines {
-		if m.State == pb.MachineMember_UP || m.State == pb.MachineMember_SUSPECT {
-			md.Append("machines", m.Machine.Id)
-		} else {
-			tui.PrintWarning(fmt.Sprintf("failed to list service containers on machine '%s' (state is %s). "+
-				"The results may be incomplete.", m.Machine.Name, m.State.String()))
-		}
-	}
-	listCtx := metadata.NewOutgoingContext(ctx, md)
-
-	// List all containers including stopped ones.
+	// Broadcast the container list request to all machines in the cluster.
+	listCtx := ProxyMachinesContext(ctx, nil)
+	// List all containers including stopped ones and deployment hooks.
 	opts := container.ListOptions{All: true}
 	machineContainers, err := cli.Docker.ListServiceContainers(listCtx, "", opts)
 	if err != nil {
 		return nil, fmt.Errorf("list containers: %w", err)
 	}
 
-	// TODO: optimise by extracting services from the list of all containers instead of inspecting each service.
-	//  Most of the code can be reused in both InspectService and ListServices.
+	// Group containers by service ID using the broadcast results.
 	servicesByID := make(map[string]api.Service)
 	for _, mc := range machineContainers {
-		// NOTE: Metadata should never be nil in practice. This is legacy fallback that will be removed.
-		if mc.Metadata == nil {
-			tui.PrintWarning("metadata is missing in response from unknown server")
-			continue
-		}
-
 		if mc.Metadata.Error != "" {
 			// TODO: return failed machines in the response.
 			tui.PrintWarning(fmt.Sprintf("failed to list containers on machine '%s': %s",
@@ -362,25 +341,30 @@ func (cli *Client) ListServices(ctx context.Context) ([]api.Service, error) {
 		}
 
 		for _, ctr := range append(mc.Containers, mc.HookContainers...) {
-			if _, ok := servicesByID[ctr.ServiceID()]; ok {
-				continue
-			}
-
-			svc, err := cli.InspectService(ctx, ctr.ServiceID())
-			if err != nil {
-				if errors.Is(err, api.ErrNotFound) {
-					continue
+			serviceID := ctr.ServiceID()
+			svc, ok := servicesByID[serviceID]
+			if !ok {
+				svc = api.Service{
+					ID:   serviceID,
+					Name: ctr.ServiceName(),
+					Mode: ctr.ServiceMode(),
 				}
-				return nil, fmt.Errorf("inspect service: %w", err)
 			}
 
-			servicesByID[ctr.ServiceID()] = svc
+			machineContainer := api.MachineServiceContainer{
+				MachineID:   mc.Metadata.MachineId,
+				MachineName: mc.Metadata.MachineName,
+				Container:   ctr,
+			}
+			if ctr.IsHook() {
+				svc.HookContainers = append(svc.HookContainers, machineContainer)
+			} else {
+				svc.Containers = append(svc.Containers, machineContainer)
+			}
+
+			servicesByID[serviceID] = svc
 		}
 	}
 
-	services := make([]api.Service, 0, len(servicesByID))
-	for _, svc := range servicesByID {
-		services = append(services, svc)
-	}
-	return services, nil
+	return slices.Collect(maps.Values(servicesByID)), nil
 }

website/docs/1-overview.md

@@ -20,7 +20,7 @@ traditional container orchestrators like Kubernetes or Swarm:
 * Scaling services across multiple machines
 * Cross-machine service communication without exposing ports to the internet
 * DNS-based service discovery
-* Automatic HTTPS and reverse proxy configuration
+* Automatic HTTPS and reverse proxy configuration with clustered storage for certificates
 * Load balancing
 * Persistent storage
 

website/docs/3-concepts/2-ingress/1-overview.md

@@ -24,3 +24,11 @@ When you [publish a service port](2-publishing-services.md), Uncloud automatical
 
 For advanced use cases, Uncloud allows to customise the Caddy config using the `x-caddy` extension in Compose files.
 See [Custom Caddy configuration](2-publishing-services.md#custom-caddy-configuration) for details.
+
+:::tip Shared TLS certificates
+
+Uncloud also provides native cluster storage for Caddy. It shares TLS certificates and ACME challenge tokens across
+machines and coordinates certificate issuance. See [Cluster storage for Caddy](4-cluster-storage-for-caddy.md)
+to enable it.
+
+:::

website/docs/3-concepts/2-ingress/3-managing-caddy.md

@@ -3,6 +3,10 @@
 Caddy is automatically deployed as a global service `caddy` when you initialise a cluster with `uc machine init`. By
 default, it runs on every machine to handle incoming HTTP/HTTPS traffic and route it to your services.
 
+Since [v0.21.0](https://github.com/psviderski/uncloud/releases/tag/v0.21.0), Uncloud provides native
+[cluster storage for Caddy](4-cluster-storage-for-caddy.md) (opt-in) to share TLS certificates and coordinate
+certificate issuance across machines.
+
 ## Checking status
 
 View the `caddy` service status and which machines it's running on:
@@ -34,7 +38,7 @@ uc caddy deploy
 Deploy a specific version or custom image:
 
 ```shell
-uc caddy deploy --image caddybuilds/caddy-cloudflare:2.10.2
+uc caddy deploy --image caddybuilds/caddy-cloudflare:2.11.4
 ```
 
 Deploy only to a specific machine or a subset of machines (comma-separated list):
@@ -80,19 +84,27 @@ config that uses the DNS challenge with Cloudflare to obtain a wildcard TLS cert
 ```yaml
 services:
   caddy:
-    image: caddybuilds/caddy-cloudflare:2.10.2
-    command: caddy run -c /config/Caddyfile
+    image: caddybuilds/caddy-cloudflare:2.11.4
     environment:
       # unix// is not a typo. Caddy uses network/address format, not a unix:// URL.
       CADDY_ADMIN: unix//run/caddy/admin.sock
     env_file:
       # Contains CLOUDFLARE_API_TOKEN=xxxxx
       - .env.secrets
+    healthcheck:
+      test: curl -fsS -o /dev/null --unix-socket /run/caddy/admin.sock http://localhost/config/
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+      start_interval: 1s
     volumes:
+      - /var/lib/uncloud/caddy:/etc/caddy:ro
+      # Persists TLS certificates and other assets when using Caddy's default local storage.
       - /var/lib/uncloud/caddy:/data
-      - /var/lib/uncloud/caddy:/config
       - /run/uncloud/caddy:/run/caddy
-      # Required by caddy.storage.uncloud module. Remove this mount if Uncloud cluster storage is not used for Caddy.
+      # Required by the Uncloud cluster storage module (https://github.com/unlabs-dev/caddy-uncloud).
+      # Remove this mount if cluster storage is not used for Caddy.
       # Mount the directory, not the socket file, so Caddy sees a replacement socket after the daemon restarts.
       - /run/uncloud/api:/run/uncloud/api:ro
     x-ports:
@@ -142,9 +154,15 @@ internal.example.com {
 
 :::info note
 
-The specified `command`, `environment`, `volumes`, and `x-ports` properties are essential for Caddy to function
-correctly in the Uncloud cluster. Do not change the source paths of the volume mounts as the Uncloud daemon relies on
-them to communicate with Caddy and update its configuration.
+The specified `environment`, `volumes`, and `x-ports` properties are essential for Caddy to function correctly in the
+Uncloud cluster. Do not change the source paths of the volume mounts as the Uncloud daemon relies on them to communicate
+with Caddy and update its configuration.
+
+The image must include `curl` for the healthcheck and start Caddy with `/etc/caddy/Caddyfile`. Images based on the
+official [Caddy image](https://hub.docker.com/_/caddy) do both by default.
+
+Keep the `/data` mount while using Caddy's default local storage so TLS certificates survive container updates. You can
+remove this mount when using [cluster storage for Caddy](4-cluster-storage-for-caddy.md).
 
 :::
 

website/docs/3-concepts/2-ingress/4-cluster-storage-for-caddy.md

@@ -0,0 +1,58 @@
+# Cluster storage for Caddy
+
+The [Uncloud storage module](https://github.com/unlabs-dev/caddy-uncloud) lets Caddy instances share TLS certificates,
+private keys, and ACME challenge tokens through Uncloud's cluster store. It uses distributed locks to coordinate
+certificate issuance. Once one instance obtains a certificate, the others can use it too.
+
+## Why use cluster storage?
+
+By default, each Caddy instance stores its certificates locally. When DNS points to multiple machines or a load balancer
+distributes traffic between them, an ACME challenge can reach a different instance from the one requesting the
+certificate. That instance may not have the challenge token, which can delay or prevent certificate issuance.
+
+Cluster storage is optional. The default Caddy image does not include the module, so you need to deploy an image that
+includes it and configure Caddy to use it.
+
+## Enabling cluster storage
+
+:::info Requirements
+
+Cluster storage requires Uncloud **v0.21.0 or newer** for both the `uc` CLI and the daemon on every cluster machine.
+Check your CLI version with `uc version` and daemon versions with `uc machine ls`. Upgrade older versions before
+enabling cluster storage.
+
+:::
+
+Create a global Caddyfile, or add `storage uncloud` to the global options in your existing one:
+
+```caddyfile title="global.Caddyfile"
+{
+    storage uncloud
+
+    # Uncomment to enable debug logs useful for troubleshooting storage operations:
+    # debug
+}
+```
+
+Deploy Caddy with the pre-built module image and your global config:
+
+```shell
+uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.1 --caddyfile global.Caddyfile
+```
+
+See the [module README](https://github.com/unlabs-dev/caddy-uncloud#usage) for custom image builds, Compose deployment,
+and additional storage options.
+
+## Verifying storage
+
+Check that `storage uncloud` appears in the Caddy config:
+
+```shell
+uc caddy config
+```
+
+List issued certificates in cluster storage with [`uc caddy cert ls`](../../9-cli-reference/uc_caddy_cert_ls.md):
+
+```shell
+uc caddy cert ls
+```

website/docs/9-cli-reference/uc_caddy.md

@@ -20,6 +20,7 @@ Manage Caddy reverse proxy service.
 ## See also
 
 * [uc](uc.md)	 - A CLI tool for managing Uncloud resources such as machines, services, and volumes.
+* [uc caddy cert](uc_caddy_cert.md)	 - Inspect certificates in cluster storage for Caddy.
 * [uc caddy config](uc_caddy_config.md)	 - Show the current Caddy configuration (Caddyfile).
 * [uc caddy deploy](uc_caddy_deploy.md)	 - Deploy or upgrade Caddy reverse proxy across all machines in the cluster.
 * [uc caddy logs](uc_caddy_logs.md)	 - View caddy logs.

website/docs/9-cli-reference/uc_caddy_cert.md

@@ -0,0 +1,24 @@
+# uc caddy cert
+
+Inspect certificates in cluster storage for Caddy.
+
+## Options
+
+```
+  -h, --help   help for cert
+```
+
+## Options inherited from parent commands
+
+```
+      --connect string          Connect to a remote cluster machine without using the Uncloud configuration file. [$UNCLOUD_CONNECT]
+                                Format: [ssh://]user@host[:port], ssh+go://user@host[:port], tcp://host:port, or unix:///path/to/uncloud.sock
+  -c, --context string          Name of the cluster context to use (default is the current context). [$UNCLOUD_CONTEXT]
+      --uncloud-config string   Path to the Uncloud configuration file. [$UNCLOUD_CONFIG] (default "~/.config/uncloud/config.yaml")
+```
+
+## See also
+
+* [uc caddy](uc_caddy.md)	 - Manage Caddy reverse proxy service.
+* [uc caddy cert ls](uc_caddy_cert_ls.md)	 - List certificates in cluster storage for Caddy.
+

website/docs/9-cli-reference/uc_caddy_cert_ls.md

@@ -0,0 +1,38 @@
+# uc caddy cert ls
+
+List certificates in cluster storage for Caddy.
+
+## Synopsis
+
+List certificates stored in the Uncloud cluster storage for Caddy.
+
+Caddy must use the Uncloud storage module (https://github.com/unlabs-dev/caddy-uncloud)
+configured with 'storage uncloud' in the global options for its certificates to appear here.
+
+This inventory does not check whether Caddy currently serves or trusts a certificate.
+
+```
+uc caddy cert ls [flags]
+```
+
+## Options
+
+```
+  -h, --help             help for ls
+  -m, --machine string   Name or ID of the machine to read certificate storage from. (default is connected machine)
+  -o, --output string    Output format: 'json' or empty for a human-readable table.
+```
+
+## Options inherited from parent commands
+
+```
+      --connect string          Connect to a remote cluster machine without using the Uncloud configuration file. [$UNCLOUD_CONNECT]
+                                Format: [ssh://]user@host[:port], ssh+go://user@host[:port], tcp://host:port, or unix:///path/to/uncloud.sock
+  -c, --context string          Name of the cluster context to use (default is the current context). [$UNCLOUD_CONTEXT]
+      --uncloud-config string   Path to the Uncloud configuration file. [$UNCLOUD_CONFIG] (default "~/.config/uncloud/config.yaml")
+```
+
+## See also
+
+* [uc caddy cert](uc_caddy_cert.md)	 - Inspect certificates in cluster storage for Caddy.
+