chore(.editorconfig): ignore formatting for release notes

Signed-off-by: Miek Gieben <miek@miek.nl>

.editorconfig

@@ -18,3 +18,6 @@ indent_size = 4
 [*.sh]
 indent_style = space
 indent_size = 4
+
+[release-notes-*.md]
+max_line_length = off

README.md

@@ -44,6 +44,8 @@ complexity of Kubernetes.
   [Uncloud DNS](https://github.com/psviderski/uncloud-dns) service.
 * **Automatic HTTPS**: Built-in Caddy reverse proxy handles TLS certificate provisioning and renewal using Let's
   Encrypt.
+* **Cluster storage for TLS certificates**: Native cluster storage for Caddy shares certificates and ACME challenge
+  tokens across machines and coordinates issuance.
 * **Docker-like CLI**: Familiar commands for managing both infrastructure and applications.
 * **Remote management**: Control your entire infrastructure through SSH access to any single machine in the cluster.
 

api/pb/caddy_storage.pb.go

@@ -446,6 +446,120 @@ func (x *StatCaddyStorageResponse) GetIsTerminal() bool {
 	return false
 }
 
+type ListCertificatesResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Certificates []*IssuedCertificate `protobuf:"bytes,1,rep,name=certificates,proto3" json:"certificates,omitempty"`
+}
+
+func (x *ListCertificatesResponse) Reset() {
+	*x = ListCertificatesResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_api_pb_caddy_storage_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ListCertificatesResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ListCertificatesResponse) ProtoMessage() {}
+
+func (x *ListCertificatesResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_api_pb_caddy_storage_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ListCertificatesResponse.ProtoReflect.Descriptor instead.
+func (*ListCertificatesResponse) Descriptor() ([]byte, []int) {
+	return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{8}
+}
+
+func (x *ListCertificatesResponse) GetCertificates() []*IssuedCertificate {
+	if x != nil {
+		return x.Certificates
+	}
+	return nil
+}
+
+type IssuedCertificate struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Subject Alternative Name (SAN) of the certificate. Caddy doesn't issue certificates with multiple SANs.
+	San string `protobuf:"bytes,1,opt,name=san,proto3" json:"san,omitempty"`
+	// The PEM-encoding of DER-encoded ASN.1 data for the cert or chain, leaf first.
+	Chain []byte `protobuf:"bytes,2,opt,name=chain,proto3" json:"chain,omitempty"`
+	// Any extra information associated with the certificate, usually provided by the issuer implementation.
+	// JSON-encoded, may be absent or null.
+	IssuerData []byte `protobuf:"bytes,3,opt,name=issuer_data,json=issuerData,proto3" json:"issuer_data,omitempty"`
+}
+
+func (x *IssuedCertificate) Reset() {
+	*x = IssuedCertificate{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_api_pb_caddy_storage_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *IssuedCertificate) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*IssuedCertificate) ProtoMessage() {}
+
+func (x *IssuedCertificate) ProtoReflect() protoreflect.Message {
+	mi := &file_api_pb_caddy_storage_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use IssuedCertificate.ProtoReflect.Descriptor instead.
+func (*IssuedCertificate) Descriptor() ([]byte, []int) {
+	return file_api_pb_caddy_storage_proto_rawDescGZIP(), []int{9}
+}
+
+func (x *IssuedCertificate) GetSan() string {
+	if x != nil {
+		return x.San
+	}
+	return ""
+}
+
+func (x *IssuedCertificate) GetChain() []byte {
+	if x != nil {
+		return x.Chain
+	}
+	return nil
+}
+
+func (x *IssuedCertificate) GetIssuerData() []byte {
+	if x != nil {
+		return x.IssuerData
+	}
+	return nil
+}
+
 var File_api_pb_caddy_storage_proto protoreflect.FileDescriptor
 
 var file_api_pb_caddy_storage_proto_rawDesc = []byte{
@@ -492,32 +606,48 @@ var file_api_pb_caddy_storage_proto_rawDesc = []byte{
 	0x65, 0x64, 0x41, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01,
 	0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x73, 0x5f, 0x74,
 	0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69,
-	0x73, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x32, 0xdf, 0x02, 0x0a, 0x0c, 0x43, 0x61,
+	0x73, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x22, 0x56, 0x0a, 0x18, 0x4c, 0x69, 0x73,
-	0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x3e, 0x0a, 0x05, 0x53, 0x74,
+	0x74, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73,
-	0x6f, 0x72, 0x65, 0x12, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x0c, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69,
+	0x63, 0x61, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70,
+	0x69, 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63,
+	0x61, 0x74, 0x65, 0x52, 0x0c, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65,
+	0x73, 0x22, 0x5c, 0x0a, 0x11, 0x49, 0x73, 0x73, 0x75, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69,
+	0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x61, 0x6e, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x61, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69,
+	0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x12, 0x1f,
+	0x0a, 0x0b, 0x69, 0x73, 0x73, 0x75, 0x65, 0x72, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x0c, 0x52, 0x0a, 0x69, 0x73, 0x73, 0x75, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x32,
+	0xaa, 0x03, 0x0a, 0x0c, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65,
+	0x12, 0x3e, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e,
+	0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67,
+	0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
+	0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
+	0x12, 0x43, 0x0a, 0x04, 0x4c, 0x6f, 0x61, 0x64, 0x12, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c,
+	0x6f, 0x61, 0x64, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x6f, 0x61,
+	0x64, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12,
+	0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x61, 0x64, 0x64,
+	0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
+	0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12,
+	0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53,
+	0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e,
+	0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f,
+	0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x04,
+	0x53, 0x74, 0x61, 0x74, 0x12, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x43,
 	0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x43, 0x61, 0x64,
-	0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x04, 0x4c, 0x6f,
+	0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x61, 0x64, 0x12, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x61, 0x64,
+	0x65, 0x12, 0x49, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69,
-	0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x63, 0x61, 0x74, 0x65, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
-	0x1a, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x61, 0x64, 0x64, 0x79,
+	0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e,
-	0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
+	0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63,
-	0x40, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e,
+	0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x26, 0x5a, 0x24,
-	0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61,
+	0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64,
-	0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
+	0x65, 0x72, 0x73, 0x6b, 0x69, 0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x61, 0x70,
-	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74,
+	0x69, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-	0x79, 0x12, 0x43, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e,
-	0x4c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69,
-	0x73, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, 0x12, 0x1c,
-	0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74,
-	0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x61,
-	0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x53, 0x74, 0x6f, 0x72,
-	0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x26, 0x5a, 0x24, 0x67,
-	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65,
-	0x72, 0x73, 0x6b, 0x69, 0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x61, 0x70, 0x69,
-	0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -532,7 +662,7 @@ func file_api_pb_caddy_storage_proto_rawDescGZIP() []byte {
 	return file_api_pb_caddy_storage_proto_rawDescData
 }
 
-var file_api_pb_caddy_storage_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
+var file_api_pb_caddy_storage_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
 var file_api_pb_caddy_storage_proto_goTypes = []any{
 	(*StoreCaddyStorageRequest)(nil),  // 0: api.StoreCaddyStorageRequest
 	(*LoadCaddyStorageRequest)(nil),   // 1: api.LoadCaddyStorageRequest
@@ -542,27 +672,32 @@ var file_api_pb_caddy_storage_proto_goTypes = []any{
 	(*ListCaddyStorageResponse)(nil),  // 5: api.ListCaddyStorageResponse
 	(*StatCaddyStorageRequest)(nil),   // 6: api.StatCaddyStorageRequest
 	(*StatCaddyStorageResponse)(nil),  // 7: api.StatCaddyStorageResponse
-	(*timestamppb.Timestamp)(nil),     // 8: google.protobuf.Timestamp
+	(*ListCertificatesResponse)(nil),  // 8: api.ListCertificatesResponse
-	(*emptypb.Empty)(nil),             // 9: google.protobuf.Empty
+	(*IssuedCertificate)(nil),         // 9: api.IssuedCertificate
+	(*timestamppb.Timestamp)(nil),     // 10: google.protobuf.Timestamp
+	(*emptypb.Empty)(nil),             // 11: google.protobuf.Empty
 }
 var file_api_pb_caddy_storage_proto_depIdxs = []int32{
-	8, // 0: api.LoadCaddyStorageResponse.updated_at:type_name -> google.protobuf.Timestamp
+	10, // 0: api.LoadCaddyStorageResponse.updated_at:type_name -> google.protobuf.Timestamp
-	8, // 1: api.StatCaddyStorageResponse.updated_at:type_name -> google.protobuf.Timestamp
+	10, // 1: api.StatCaddyStorageResponse.updated_at:type_name -> google.protobuf.Timestamp
-	0, // 2: api.CaddyStorage.Store:input_type -> api.StoreCaddyStorageRequest
+	9,  // 2: api.ListCertificatesResponse.certificates:type_name -> api.IssuedCertificate
-	1, // 3: api.CaddyStorage.Load:input_type -> api.LoadCaddyStorageRequest
+	0,  // 3: api.CaddyStorage.Store:input_type -> api.StoreCaddyStorageRequest
-	3, // 4: api.CaddyStorage.Delete:input_type -> api.DeleteCaddyStorageRequest
+	1,  // 4: api.CaddyStorage.Load:input_type -> api.LoadCaddyStorageRequest
-	4, // 5: api.CaddyStorage.List:input_type -> api.ListCaddyStorageRequest
+	3,  // 5: api.CaddyStorage.Delete:input_type -> api.DeleteCaddyStorageRequest
-	6, // 6: api.CaddyStorage.Stat:input_type -> api.StatCaddyStorageRequest
+	4,  // 6: api.CaddyStorage.List:input_type -> api.ListCaddyStorageRequest
-	9, // 7: api.CaddyStorage.Store:output_type -> google.protobuf.Empty
+	6,  // 7: api.CaddyStorage.Stat:input_type -> api.StatCaddyStorageRequest
-	2, // 8: api.CaddyStorage.Load:output_type -> api.LoadCaddyStorageResponse
+	11, // 8: api.CaddyStorage.ListCertificates:input_type -> google.protobuf.Empty
-	9, // 9: api.CaddyStorage.Delete:output_type -> google.protobuf.Empty
+	11, // 9: api.CaddyStorage.Store:output_type -> google.protobuf.Empty
-	5, // 10: api.CaddyStorage.List:output_type -> api.ListCaddyStorageResponse
+	2,  // 10: api.CaddyStorage.Load:output_type -> api.LoadCaddyStorageResponse
-	7, // 11: api.CaddyStorage.Stat:output_type -> api.StatCaddyStorageResponse
+	11, // 11: api.CaddyStorage.Delete:output_type -> google.protobuf.Empty
-	7, // [7:12] is the sub-list for method output_type
+	5,  // 12: api.CaddyStorage.List:output_type -> api.ListCaddyStorageResponse
-	2, // [2:7] is the sub-list for method input_type
+	7,  // 13: api.CaddyStorage.Stat:output_type -> api.StatCaddyStorageResponse
-	2, // [2:2] is the sub-list for extension type_name
+	8,  // 14: api.CaddyStorage.ListCertificates:output_type -> api.ListCertificatesResponse
-	2, // [2:2] is the sub-list for extension extendee
+	9,  // [9:15] is the sub-list for method output_type
-	0, // [0:2] is the sub-list for field type_name
+	3,  // [3:9] is the sub-list for method input_type
+	3,  // [3:3] is the sub-list for extension type_name
+	3,  // [3:3] is the sub-list for extension extendee
+	0,  // [0:3] is the sub-list for field type_name
 }
 
 func init() { file_api_pb_caddy_storage_proto_init() }
@@ -667,6 +802,30 @@ func file_api_pb_caddy_storage_proto_init() {
 				return nil
 			}
 		}
+		file_api_pb_caddy_storage_proto_msgTypes[8].Exporter = func(v any, i int) any {
+			switch v := v.(*ListCertificatesResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_api_pb_caddy_storage_proto_msgTypes[9].Exporter = func(v any, i int) any {
+			switch v := v.(*IssuedCertificate); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -674,7 +833,7 @@ func file_api_pb_caddy_storage_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_api_pb_caddy_storage_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   8,
+			NumMessages:   10,
 			NumExtensions: 0,
 			NumServices:   1,
 		},

api/pb/caddy_storage.proto

@@ -18,6 +18,10 @@ service CaddyStorage {
   rpc Delete(DeleteCaddyStorageRequest) returns (google.protobuf.Empty);
   rpc List(ListCaddyStorageRequest) returns (ListCaddyStorageResponse);
   rpc Stat(StatCaddyStorageRequest) returns (StatCaddyStorageResponse);
+
+  // ListCertificates lists certificates issued by Caddy and stored in the cluster store.
+  // Unreadable entries and invalid resource metadata are skipped. Chains and issuer data are returned as stored.
+  rpc ListCertificates(google.protobuf.Empty) returns (ListCertificatesResponse);
 }
 
 message StoreCaddyStorageRequest {
@@ -57,3 +61,17 @@ message StatCaddyStorageResponse {
   int64 size = 3;
   bool is_terminal = 4;
 }
+
+message ListCertificatesResponse {
+  repeated IssuedCertificate certificates = 1;
+}
+
+message IssuedCertificate {
+  // Subject Alternative Name (SAN) of the certificate. Caddy doesn't issue certificates with multiple SANs.
+  string san = 1;
+  // The PEM-encoding of DER-encoded ASN.1 data for the cert or chain, leaf first.
+  bytes chain = 2;
+  // Any extra information associated with the certificate, usually provided by the issuer implementation.
+  // JSON-encoded, may be absent or null.
+  bytes issuer_data = 3;
+}

api/pb/caddy_storage_grpc.pb.go

@@ -20,11 +20,12 @@ import (
 const _ = grpc.SupportPackageIsVersion9
 
 const (
-	CaddyStorage_Store_FullMethodName  = "/api.CaddyStorage/Store"
+	CaddyStorage_Store_FullMethodName            = "/api.CaddyStorage/Store"
-	CaddyStorage_Load_FullMethodName   = "/api.CaddyStorage/Load"
+	CaddyStorage_Load_FullMethodName             = "/api.CaddyStorage/Load"
-	CaddyStorage_Delete_FullMethodName = "/api.CaddyStorage/Delete"
+	CaddyStorage_Delete_FullMethodName           = "/api.CaddyStorage/Delete"
-	CaddyStorage_List_FullMethodName   = "/api.CaddyStorage/List"
+	CaddyStorage_List_FullMethodName             = "/api.CaddyStorage/List"
-	CaddyStorage_Stat_FullMethodName   = "/api.CaddyStorage/Stat"
+	CaddyStorage_Stat_FullMethodName             = "/api.CaddyStorage/Stat"
+	CaddyStorage_ListCertificates_FullMethodName = "/api.CaddyStorage/ListCertificates"
 )
 
 // CaddyStorageClient is the client API for CaddyStorage service.
@@ -42,6 +43,9 @@ type CaddyStorageClient interface {
 	Delete(ctx context.Context, in *DeleteCaddyStorageRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
 	List(ctx context.Context, in *ListCaddyStorageRequest, opts ...grpc.CallOption) (*ListCaddyStorageResponse, error)
 	Stat(ctx context.Context, in *StatCaddyStorageRequest, opts ...grpc.CallOption) (*StatCaddyStorageResponse, error)
+	// ListCertificates lists certificates issued by Caddy and stored in the cluster store.
+	// Unreadable entries and invalid resource metadata are skipped. Chains and issuer data are returned as stored.
+	ListCertificates(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ListCertificatesResponse, error)
 }
 
 type caddyStorageClient struct {
@@ -102,6 +106,16 @@ func (c *caddyStorageClient) Stat(ctx context.Context, in *StatCaddyStorageReque
 	return out, nil
 }
 
+func (c *caddyStorageClient) ListCertificates(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ListCertificatesResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ListCertificatesResponse)
+	err := c.cc.Invoke(ctx, CaddyStorage_ListCertificates_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // CaddyStorageServer is the server API for CaddyStorage service.
 // All implementations must embed UnimplementedCaddyStorageServer
 // for forward compatibility.
@@ -117,6 +131,9 @@ type CaddyStorageServer interface {
 	Delete(context.Context, *DeleteCaddyStorageRequest) (*emptypb.Empty, error)
 	List(context.Context, *ListCaddyStorageRequest) (*ListCaddyStorageResponse, error)
 	Stat(context.Context, *StatCaddyStorageRequest) (*StatCaddyStorageResponse, error)
+	// ListCertificates lists certificates issued by Caddy and stored in the cluster store.
+	// Unreadable entries and invalid resource metadata are skipped. Chains and issuer data are returned as stored.
+	ListCertificates(context.Context, *emptypb.Empty) (*ListCertificatesResponse, error)
 	mustEmbedUnimplementedCaddyStorageServer()
 }
 
@@ -142,6 +159,9 @@ func (UnimplementedCaddyStorageServer) List(context.Context, *ListCaddyStorageRe
 func (UnimplementedCaddyStorageServer) Stat(context.Context, *StatCaddyStorageRequest) (*StatCaddyStorageResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Stat not implemented")
 }
+func (UnimplementedCaddyStorageServer) ListCertificates(context.Context, *emptypb.Empty) (*ListCertificatesResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ListCertificates not implemented")
+}
 func (UnimplementedCaddyStorageServer) mustEmbedUnimplementedCaddyStorageServer() {}
 func (UnimplementedCaddyStorageServer) testEmbeddedByValue()                      {}
 
@@ -253,6 +273,24 @@ func _CaddyStorage_Stat_Handler(srv interface{}, ctx context.Context, dec func(i
 	return interceptor(ctx, in, info, handler)
 }
 
+func _CaddyStorage_ListCertificates_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(emptypb.Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(CaddyStorageServer).ListCertificates(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: CaddyStorage_ListCertificates_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(CaddyStorageServer).ListCertificates(ctx, req.(*emptypb.Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 // CaddyStorage_ServiceDesc is the grpc.ServiceDesc for CaddyStorage service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -280,6 +318,10 @@ var CaddyStorage_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "Stat",
 			Handler:    _CaddyStorage_Stat_Handler,
 		},
+		{
+			MethodName: "ListCertificates",
+			Handler:    _CaddyStorage_ListCertificates_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "api/pb/caddy_storage.proto",

api/pb/machine.proto

@@ -20,7 +20,7 @@ service Machine {
   // InspectMachine retrieves detailed information about the machine. Supports broadcasting to multiple machines.
   rpc InspectMachine(google.protobuf.Empty) returns (InspectMachineResponse);
   // WaitForStoreVersion waits until the cluster store on this machine has reached each requested actor version, with
-  // no known missing or pending transactions through those versions.
+  // no known missing or pending transactions through those versions from active members.
   // Corrosion may satisfy a version by applying its surviving changes or by marking it complete because its changes
   // have been superseded.
   //

api/pb/machine_grpc.pb.go

@@ -48,7 +48,7 @@ type MachineClient interface {
 	// InspectMachine retrieves detailed information about the machine. Supports broadcasting to multiple machines.
 	InspectMachine(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*InspectMachineResponse, error)
 	// WaitForStoreVersion waits until the cluster store on this machine has reached each requested actor version, with
-	// no known missing or pending transactions through those versions.
+	// no known missing or pending transactions through those versions from active members.
 	// Corrosion may satisfy a version by applying its surviving changes or by marking it complete because its changes
 	// have been superseded.
 	//
@@ -224,7 +224,7 @@ type MachineServer interface {
 	// InspectMachine retrieves detailed information about the machine. Supports broadcasting to multiple machines.
 	InspectMachine(context.Context, *emptypb.Empty) (*InspectMachineResponse, error)
 	// WaitForStoreVersion waits until the cluster store on this machine has reached each requested actor version, with
-	// no known missing or pending transactions through those versions.
+	// no known missing or pending transactions through those versions from active members.
 	// Corrosion may satisfy a version by applying its surviving changes or by marking it complete because its changes
 	// have been superseded.
 	//

caddystorage/locker.go

@@ -1,165 +0,0 @@
-package caddystorage
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"slices"
-	"time"
-
-	"github.com/psviderski/uncloud/pkg/client"
-	"github.com/psviderski/uncloud/pkg/distlock"
-)
-
-const (
-	lockPrefix              = "caddy_storage:"
-	storeReplicationTimeout = 10 * time.Second
-)
-
-// Lock acquires an automatically renewed distributed lock and waits for the local store to catch up with versions
-// observed on responding machines. Writers using the same lock can then read locally. Unavailable machines may have
-// writes that this wait does not cover, and reads outside a lock remain eventually consistent.
-func (s *Storage) Lock(ctx context.Context, name string) (err error) {
-	if name == "" {
-		return errors.New("lock name is empty")
-	}
-
-	// Count the call first. If Cleanup has already observed zero, Caddy has cancelled s.ctx and the check below rejects
-	// this call before it uses the client.
-	s.lockOps.Add(1)
-	// A failed Lock owns its lifecycle through any lease rollback. A successful Lock transfers that responsibility to
-	// Unlock, which keeps the client open until its release attempt finishes.
-	defer func() {
-		if err != nil {
-			s.lockOps.Add(-1)
-		}
-	}()
-	if s.ctx.Err() != nil {
-		return errors.New("storage is closed")
-	}
-
-	// Stop acquisition retries when Caddy unloads the module (cancels s.ctx), even if the caller's context
-	// is still active.
-	ctx, cancel := context.WithCancelCause(ctx)
-	defer cancel(nil)
-	stopOnCleanup := context.AfterFunc(s.ctx, func() {
-		cancel(context.Cause(s.ctx))
-	})
-	defer stopOnCleanup()
-
-	log := s.log.With("lock", name)
-	started := time.Now()
-	log.Debug("acquiring lock", "ttl", time.Duration(s.LockTTL))
-	lease, err := s.locker.Acquire(ctx, lockPrefix+name)
-	if err != nil {
-		log.Debug("failed to acquire lock", "duration", time.Since(started), "error", err)
-		return fmt.Errorf("acquire lock '%s': %w", name, err)
-	}
-	log.Debug("lock lease acquired", "duration", time.Since(started))
-	// Release the lease if the lock acquisition fails after this point.
-	// Unlock will take care of releasing the lease on success.
-	defer func() {
-		if err == nil {
-			return
-		}
-		if releaseErr := s.releaseLock(ctx, name, lease, "cancelled acquisition"); releaseErr != nil {
-			err = errors.Join(err, releaseErr)
-		}
-	}()
-
-	// Catch up with the latest store versions observed on responding machines to increase the chance of reading
-	// the latest writes on them locally.
-	version, machines, err := s.clusterStoreVersion(ctx, log)
-	if err != nil {
-		return err
-	}
-	waitStarted := time.Now()
-	log.Debug("waiting for local store replication", "machine_names", machines, "store_version", version)
-	waitCtx, cancelWait := context.WithTimeout(ctx, storeReplicationTimeout)
-	err = s.client.WaitForStoreVersion(waitCtx, version)
-	cancelWait()
-	if err != nil {
-		return fmt.Errorf("wait for local store replication: %w", err)
-	}
-	log.Debug("local store replication complete", "duration", time.Since(waitStarted))
-
-	s.locksMu.Lock()
-	defer s.locksMu.Unlock()
-	if s.ctx.Err() != nil {
-		return errors.New("storage is closed")
-	}
-	if lost := context.Cause(lease.Context()); lost != nil {
-		return lost
-	}
-	if _, exists := s.locks[name]; exists {
-		return errors.New("lock is already tracked by this storage instance")
-	}
-	s.locks[name] = lease
-
-	log.Debug("lock acquired", "duration", time.Since(started))
-	return nil
-}
-
-// clusterStoreVersion returns the per-actor maximum store versions from responding machines and their names.
-func (s *Storage) clusterStoreVersion(ctx context.Context, log *slog.Logger) (map[string]uint64, []string, error) {
-	ctx, cancel := context.WithTimeout(ctx, distlock.DefaultMaxNodeCallTimeout)
-	defer cancel()
-	resp, err := s.client.MachineClient.InspectMachine(client.ProxyMachinesContext(ctx, nil), nil)
-	if err != nil {
-		return nil, nil, fmt.Errorf("inspect machines for store versions: %w", err)
-	}
-
-	maxVersion := make(map[string]uint64)
-	machines := make([]string, 0, len(resp.Machines))
-	for _, m := range resp.Machines {
-		if m.Metadata.Error != "" {
-			log.Warn("skipping machine when collecting store versions",
-				"id", m.Metadata.MachineId, "name", m.Metadata.MachineName, "error", m.Metadata.Error)
-			continue
-		}
-		machines = append(machines, m.Metadata.MachineName)
-		for actor, v := range m.StoreVersion {
-			maxVersion[actor] = max(maxVersion[actor], v)
-		}
-	}
-	slices.Sort(machines)
-	return maxVersion, machines, nil
-}
-
-// Unlock releases a previously acquired distributed lock.
-func (s *Storage) Unlock(ctx context.Context, name string) error {
-	s.locksMu.Lock()
-	lease, exists := s.locks[name]
-	if exists {
-		delete(s.locks, name)
-	}
-	s.locksMu.Unlock()
-	if !exists {
-		return fmt.Errorf("lock '%s' is not held by this storage instance", name)
-	}
-	defer s.lockOps.Add(-1)
-
-	// Release stops renewal even on error. Any nodes that cannot be reached will let the lease expire.
-	if err := s.releaseLock(ctx, name, lease, "unlock"); err != nil {
-		return fmt.Errorf("release lock '%s': %w", name, err)
-	}
-
-	return nil
-}
-
-// releaseLock releases the lease with a context that ignores cancellation and logs the release attempt.
-func (s *Storage) releaseLock(ctx context.Context, name string, lease *distlock.Lease, reason string) error {
-	// Unlock and rollback must attempt node cleanup even if the caller or module has already been cancelled.
-	ctx = context.WithoutCancel(ctx)
-
-	log := s.log.With("lock", name, "reason", reason)
-	started := time.Now()
-	log.Debug("releasing lock")
-	if err := lease.Release(ctx); err != nil {
-		log.Debug("failed to release lock", "duration", time.Since(started), "error", err)
-		return err
-	}
-	log.Debug("lock released", "duration", time.Since(started))
-	return nil
-}

caddystorage/module.go

@@ -1,211 +0,0 @@
-// Package caddystorage provides Caddy storage backed by an Uncloud cluster.
-package caddystorage
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/caddyserver/caddy/v2"
-	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
-	"github.com/caddyserver/certmagic"
-	"github.com/psviderski/uncloud/pkg/client"
-	"github.com/psviderski/uncloud/pkg/client/connector"
-	"github.com/psviderski/uncloud/pkg/distlock"
-)
-
-const (
-	// ModuleID is the Caddy module ID for Uncloud storage.
-	ModuleID = "caddy.storage.uncloud"
-	// DefaultSocketPath is the default path to the Uncloud API socket.
-	DefaultSocketPath = "/run/uncloud/api/uncloud.sock"
-	// DefaultLockTTL is the default duration of a distributed lock lease.
-	DefaultLockTTL = 20 * time.Second
-	// lockCleanupTimeout bounds how long an unloaded module waits for active lock operations when cleaning up.
-	lockCleanupTimeout = 5 * time.Minute
-)
-
-//nolint:gochecknoinits // Caddy modules must register during package initialisation.
-func init() {
-	caddy.RegisterModule(new(Storage))
-}
-
-// Storage implements a Caddy storage backend that uses an Uncloud cluster to store assets such as TLS certificates.
-type Storage struct {
-	// Socket is the path to the Uncloud API socket.
-	// Defaults to /run/uncloud/api/uncloud.sock when not set.
-	Socket string `json:"socket,omitempty"`
-	// LockTTL is the duration of a distributed lock after which it expires if not renewed. Locks renew automatically
-	// until unlocked. If an instance crashes or cannot renew, expiry allows another instance to acquire the stale lock.
-	// Longer durations tolerate longer interruptions but delay recovery after a crash. Normal unlocks release the lock
-	// immediately.
-	// Defaults to 20 seconds when not set.
-	LockTTL caddy.Duration `json:"lock_ttl,omitempty"`
-
-	client *client.Client
-	locker *distlock.Locker
-	log    *slog.Logger
-
-	// ctx is the module context from Provision. Caddy cancels it when it unloads the module and calls Cleanup.
-	ctx context.Context
-
-	// locksMu protects locks.
-	locksMu sync.Mutex
-	// locks maps successfully acquired lock names to held leases.
-	locks map[string]*distlock.Lease
-	// lockOps counts Lock calls until they fail or their locks are released.
-	lockOps atomic.Int64
-}
-
-// CaddyModule returns the Caddy module information.
-func (*Storage) CaddyModule() caddy.ModuleInfo {
-	return caddy.ModuleInfo{
-		ID:  ModuleID,
-		New: func() caddy.Module { return new(Storage) },
-	}
-}
-
-// Provision connects the storage to the local Uncloud API and initialises the distributed locker.
-func (s *Storage) Provision(ctx caddy.Context) error {
-	s.ctx = ctx
-	s.log = ctx.Slogger()
-
-	if s.Socket == "" {
-		s.Socket = DefaultSocketPath
-	}
-	if s.LockTTL == 0 {
-		s.LockTTL = caddy.Duration(DefaultLockTTL)
-	}
-	if s.LockTTL < 0 {
-		return errors.New("lock_ttl must be positive")
-	}
-
-	cli, err := client.New(ctx, connector.NewUnixConnector(s.Socket))
-	if err != nil {
-		return fmt.Errorf("connect to Uncloud API: %w", err)
-	}
-
-	locker, err := cli.NewLocker(distlock.Config{
-		LeaseDuration: time.Duration(s.LockTTL),
-	})
-	if err != nil {
-		_ = cli.Close()
-		return fmt.Errorf("create distributed locker: %w", err)
-	}
-
-	s.client = cli
-	s.locker = locker
-	s.locks = make(map[string]*distlock.Lease)
-
-	s.log.Info("module provisioned", "socket", s.Socket, "lock_ttl", time.Duration(s.LockTTL))
-	return nil
-}
-
-// Cleanup keeps the Uncloud API connection open so acquired lock leases can renew until Caddy releases them. It closes
-// the connection after all lock operations finish or the cleanup timeout expires.
-func (s *Storage) Cleanup() error {
-	if s.client == nil {
-		// Provision failed before opening the storage.
-		return nil
-	}
-
-	started := time.Now()
-	s.log.Debug("cleaning up module", "locks", s.lockOps.Load())
-
-	go func() {
-		timer := time.NewTimer(lockCleanupTimeout)
-		defer timer.Stop()
-		ticker := time.NewTicker(1 * time.Second)
-		defer ticker.Stop()
-
-		timedOut := false
-		for !timedOut && s.lockOps.Load() > 0 {
-			select {
-			case <-ticker.C:
-			case <-timer.C:
-				timedOut = true
-			}
-		}
-		if remaining := s.lockOps.Load(); timedOut && remaining > 0 {
-			s.log.Warn("timed out waiting for active locks to be unlocked",
-				"locks", remaining, "timeout", lockCleanupTimeout)
-		}
-
-		if err := s.client.Close(); err != nil {
-			s.log.Warn("failed to clean up module", "duration", time.Since(started), "error", err)
-			return
-		}
-		s.log.Debug("module cleanup complete", "duration", time.Since(started))
-	}()
-
-	return nil
-}
-
-// CertMagicStorage returns the provisioned CertMagic storage implementation.
-func (s *Storage) CertMagicStorage() (certmagic.Storage, error) {
-	return s, nil
-}
-
-// UnmarshalCaddyfile configures Uncloud storage from the Caddyfile global storage block.
-//
-//	{
-//	    storage uncloud {
-//	        socket /run/uncloud/api/uncloud.sock
-//	        lock_ttl 20s
-//	    }
-//	}
-func (s *Storage) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
-	d.Next() // Skip the module name 'uncloud'.
-	// Reject inline arguments. NextArg leaves an opening brace for NextBlock.
-	if d.NextArg() {
-		return d.ArgErr()
-	}
-
-	// Read the optional options block, skipping its surrounding braces.
-	for d.NextBlock(0) {
-		switch d.Val() {
-		case "socket":
-			// Require a socket path on the same line as 'socket' option.
-			if !d.NextArg() {
-				return d.ArgErr()
-			}
-			s.Socket = d.Val()
-			// Reject extra arguments after the socket path.
-			if d.NextArg() {
-				return d.ArgErr()
-			}
-		case "lock_ttl":
-			if !d.NextArg() {
-				return d.ArgErr()
-			}
-			ttl, err := caddy.ParseDuration(d.Val())
-			if err != nil {
-				return d.Errf("invalid lock_ttl '%s': %w", d.Val(), err)
-			}
-			if ttl <= 0 {
-				return d.Err("lock_ttl must be positive")
-			}
-			if d.NextArg() {
-				return d.ArgErr()
-			}
-			s.LockTTL = caddy.Duration(ttl)
-		default:
-			return d.Errf("unknown uncloud storage option: '%s'", d.Val())
-		}
-	}
-
-	return nil
-}
-
-var (
-	_ caddy.Module           = (*Storage)(nil)
-	_ caddy.Provisioner      = (*Storage)(nil)
-	_ caddy.CleanerUpper     = (*Storage)(nil)
-	_ caddy.StorageConverter = (*Storage)(nil)
-	_ caddyfile.Unmarshaler  = (*Storage)(nil)
-	_ certmagic.Storage      = (*Storage)(nil)
-)

caddystorage/storage.go

@@ -1,83 +0,0 @@
-package caddystorage
-
-import (
-	"context"
-	"fmt"
-	"io/fs"
-
-	"github.com/caddyserver/certmagic"
-	"github.com/psviderski/uncloud/api/pb"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
-)
-
-// Store writes a value to the cluster store.
-func (s *Storage) Store(ctx context.Context, key string, value []byte) error {
-	if _, err := s.client.CaddyStorage.Store(ctx, &pb.StoreCaddyStorageRequest{Key: key, Value: value}); err != nil {
-		return storageError("store", key, err)
-	}
-	return nil
-}
-
-// Load reads a value from the cluster store.
-func (s *Storage) Load(ctx context.Context, key string) ([]byte, error) {
-	resp, err := s.client.CaddyStorage.Load(ctx, &pb.LoadCaddyStorageRequest{Key: key})
-	if err != nil {
-		return nil, storageError("load", key, err)
-	}
-	return resp.Value, nil
-}
-
-// Delete removes a key and its descendants from the cluster store.
-func (s *Storage) Delete(ctx context.Context, key string) error {
-	if _, err := s.client.CaddyStorage.Delete(ctx, &pb.DeleteCaddyStorageRequest{Key: key}); err != nil {
-		return storageError("delete", key, err)
-	}
-	return nil
-}
-
-// Exists reports whether a key exists in the cluster store.
-func (s *Storage) Exists(ctx context.Context, key string) bool {
-	_, err := s.Stat(ctx, key)
-	return err == nil
-}
-
-// List returns keys under prefix from the cluster store.
-func (s *Storage) List(ctx context.Context, prefix string, recursive bool) ([]string, error) {
-	resp, err := s.client.CaddyStorage.List(ctx, &pb.ListCaddyStorageRequest{
-		Prefix:    prefix,
-		Recursive: recursive,
-	})
-	if err != nil {
-		return nil, storageError("list", prefix, err)
-	}
-	return resp.Keys, nil
-}
-
-// Stat returns information about a key in the cluster store.
-func (s *Storage) Stat(ctx context.Context, key string) (certmagic.KeyInfo, error) {
-	resp, err := s.client.CaddyStorage.Stat(ctx, &pb.StatCaddyStorageRequest{Key: key})
-	if err != nil {
-		return certmagic.KeyInfo{}, storageError("stat", key, err)
-	}
-
-	info := certmagic.KeyInfo{
-		Key:        resp.Key,
-		Size:       resp.Size,
-		IsTerminal: resp.IsTerminal,
-	}
-	if resp.UpdatedAt != nil {
-		if err := resp.UpdatedAt.CheckValid(); err != nil {
-			return certmagic.KeyInfo{}, storageError("stat", key, fmt.Errorf("invalid updated_at timestamp: %w", err))
-		}
-		info.Modified = resp.UpdatedAt.AsTime()
-	}
-	return info, nil
-}
-
-func storageError(operation, key string, err error) error {
-	if status.Code(err) == codes.NotFound {
-		err = fs.ErrNotExist
-	}
-	return fmt.Errorf("uncloud storage: %s key %q: %w", operation, key, err)
-}

cmd/uc/caddy/cert/ls.go

@@ -0,0 +1,167 @@
+package cert
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"charm.land/lipgloss/v2"
+	"github.com/caddyserver/certmagic"
+	"github.com/docker/go-units"
+	"github.com/psviderski/uncloud/internal/cli"
+	"github.com/psviderski/uncloud/internal/cli/completion"
+	"github.com/psviderski/uncloud/internal/cli/tui"
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/psviderski/uncloud/pkg/client"
+	"github.com/spf13/cobra"
+)
+
+type listOptions struct {
+	machine string
+	output  string
+}
+
+func NewListCommand() *cobra.Command {
+	opts := listOptions{}
+	cmd := &cobra.Command{
+		Use:     "ls",
+		Aliases: []string{"list"},
+		Short:   "List certificates in cluster storage for Caddy.",
+		Long: `List certificates stored in the Uncloud cluster storage for Caddy.
+
+Caddy must use the Uncloud storage module (https://github.com/unlabs-dev/caddy-uncloud)
+configured with 'storage uncloud' in the global options for its certificates to appear here.
+
+This inventory does not check whether Caddy currently serves or trusts a certificate.`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			uncli := cmd.Context().Value("cli").(*cli.CLI)
+			return list(cmd.Context(), uncli, opts)
+		},
+	}
+
+	cmd.Flags().StringVarP(&opts.machine, "machine", "m", "",
+		"Name or ID of the machine to read certificate storage from. (default is connected machine)")
+	cmd.Flags().StringVarP(&opts.output, "output", "o", "",
+		"Output format: 'json' or empty for a human-readable table.")
+	completion.MachinesFlag(cmd)
+
+	return cmd
+}
+
+func list(ctx context.Context, uncli *cli.CLI, opts listOptions) error {
+	if opts.output != "" && opts.output != "json" {
+		return fmt.Errorf("unsupported output format '%s' (supported: json)", opts.output)
+	}
+
+	clusterClient, err := uncli.ConnectCluster(ctx)
+	if err != nil {
+		return fmt.Errorf("connect to cluster: %w", err)
+	}
+	defer clusterClient.Close()
+
+	certs, listErr := clusterClient.Caddy.ListCertificates(ctx,
+		client.CaddyListCertificatesOptions{Machine: opts.machine})
+	if len(certs) > 0 || listErr == nil {
+		if err = printCertificates(os.Stdout, certs, opts.output); err != nil {
+			return fmt.Errorf("print Caddy certificates: %w", err)
+		}
+	}
+	if listErr != nil {
+		if len(certs) > 0 {
+			return fmt.Errorf("certificate list is partial: %w", listErr)
+		}
+		return fmt.Errorf("list Caddy certificates: %w", listErr)
+	}
+
+	return nil
+}
+
+func printCertificates(out io.Writer, certs []api.IssuedCertificate, output string) error {
+	type certificate struct {
+		api.IssuedCertificate
+		Fingerprint string
+	}
+
+	items := make([]certificate, 0, len(certs))
+	for _, cert := range certs {
+		fingerprint := cert.Fingerprint()
+		items = append(items, certificate{
+			IssuedCertificate: cert,
+			Fingerprint:       hex.EncodeToString(fingerprint[:]),
+		})
+	}
+	slices.SortFunc(items, func(a, b certificate) int {
+		if cmp := strings.Compare(a.SAN, b.SAN); cmp != 0 {
+			return cmp
+		}
+		if cmp := a.Chain[0].NotAfter.Compare(b.Chain[0].NotAfter); cmp != 0 {
+			return cmp
+		}
+		return strings.Compare(a.Fingerprint, b.Fingerprint)
+	})
+
+	if output == "json" {
+		encoder := json.NewEncoder(out)
+		encoder.SetIndent("", "  ")
+		if err := encoder.Encode(items); err != nil {
+			return fmt.Errorf("marshal certificates: %w", err)
+		}
+		return nil
+	}
+
+	if len(items) == 0 {
+		_, err := fmt.Fprintln(out, "No Caddy certificates found in cluster storage.")
+		return err
+	}
+
+	t := tui.NewTable()
+	t.Headers("ID", "NAME", "ISSUER", "EXPIRES")
+	for _, item := range items {
+		t.Row(
+			item.Fingerprint[:12],
+			item.SAN,
+			formatIssuer(item.IssuerData),
+			formatExpiry(item.Chain[0].NotAfter),
+		)
+	}
+	_, err := lipgloss.Fprintln(out, t)
+	return err
+}
+
+func formatIssuer(data api.CertificateIssuerData) string {
+	if data.ACME == nil {
+		return "unknown"
+	}
+
+	switch data.ACME.CA {
+	case certmagic.LetsEncryptProductionCA:
+		return "Let's Encrypt"
+	case certmagic.LetsEncryptStagingCA:
+		return "Let's Encrypt (staging)"
+	case certmagic.ZeroSSLProductionCA:
+		return "ZeroSSL"
+	case certmagic.GoogleTrustProductionCA:
+		return "Google Trust Services"
+	case certmagic.GoogleTrustStagingCA:
+		return "Google Trust Services (staging)"
+	default:
+		return data.ACME.CA
+	}
+}
+
+func formatExpiry(expires time.Time) string {
+	delta := expires.Sub(time.Now())
+	if delta <= 0 {
+		return fmt.Sprintf("%s (expired %s ago)", expires.UTC().Format(time.DateOnly),
+			strings.ToLower(units.HumanDuration(-delta)))
+	}
+	return fmt.Sprintf("%s (%s)", expires.UTC().Format(time.DateOnly),
+		strings.ToLower(units.HumanDuration(delta)))
+}

cmd/uc/caddy/cert/ls_test.go

@@ -0,0 +1,128 @@
+package cert
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/caddyserver/certmagic"
+	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPrintCertificates(t *testing.T) {
+	now := time.Now().UTC().Truncate(time.Hour)
+	certs := []api.IssuedCertificate{
+		testIssuedCertificate("z.example.com", "z", now.Add(48*time.Hour), certmagic.LetsEncryptProductionCA),
+		testIssuedCertificate("a.example.com", "old", now.Add(-48*time.Hour), certmagic.LetsEncryptStagingCA),
+		testIssuedCertificate("a.example.com", "new", now.Add(72*time.Hour), "https://ca.example/directory"),
+		testIssuedCertificate("unknown.example.com", "unknown", now.Add(96*time.Hour), ""),
+	}
+
+	var table bytes.Buffer
+	require.NoError(t, printCertificates(&table, certs, ""))
+	output := table.String()
+	assert.Contains(t, output, "ID")
+	assert.Contains(t, output, "NAME")
+	assert.Contains(t, output, "ISSUER")
+	assert.Contains(t, output, "EXPIRES")
+	assert.Contains(t, output, now.Add(-48*time.Hour).Format(time.DateOnly)+" (expired ")
+	assert.Contains(t, output, now.Add(48*time.Hour).Format(time.DateOnly)+" (")
+	assert.Contains(t, output, "Let's Encrypt (staging)")
+	assert.Contains(t, output, "https://ca.example/directory")
+	assert.Less(t, strings.Index(output, now.Add(-48*time.Hour).Format(time.DateOnly)),
+		strings.Index(output, now.Add(72*time.Hour).Format(time.DateOnly)))
+	assert.Less(t, strings.Index(output, "a.example.com"), strings.Index(output, "z.example.com"))
+	assert.Contains(t, output, testFingerprint("old")[:12])
+
+	var encoded bytes.Buffer
+	require.NoError(t, printCertificates(&encoded, certs, "json"))
+	var items []struct {
+		api.IssuedCertificate
+		Fingerprint string
+	}
+	require.NoError(t, json.Unmarshal(encoded.Bytes(), &items))
+	require.Len(t, items, 4)
+	assert.Equal(t, testFingerprint("old"), items[0].Fingerprint)
+	assert.Equal(t, "a.example.com", items[0].SAN)
+	require.Len(t, items[0].Chain, 1)
+	assert.Equal(t, []byte("old"), items[0].Chain[0].Raw)
+	assert.Equal(t, now.Add(-48*time.Hour), items[0].Chain[0].NotAfter)
+	require.NotNil(t, items[0].IssuerData.ACME)
+	assert.Equal(t, certmagic.LetsEncryptStagingCA, items[0].IssuerData.ACME.CA)
+	assert.Equal(t, testFingerprint("new"), items[1].Fingerprint)
+	require.NotNil(t, items[1].IssuerData.ACME)
+	assert.Equal(t, "https://ca.example/directory", items[1].IssuerData.ACME.CA)
+	assert.Nil(t, items[2].IssuerData.ACME)
+	assert.Equal(t, "z.example.com", items[3].SAN)
+}
+
+func TestPrintCertificates_Empty(t *testing.T) {
+	for _, tt := range []struct {
+		output string
+		want   string
+	}{
+		{output: "", want: "No Caddy certificates found in cluster storage.\n"},
+		{output: "json", want: "[]\n"},
+	} {
+		var out bytes.Buffer
+		require.NoError(t, printCertificates(&out, nil, tt.output))
+		assert.Equal(t, tt.want, out.String())
+	}
+}
+
+func TestFormatIssuer(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		ca   string
+		want string
+	}{
+		{name: "production", ca: certmagic.LetsEncryptProductionCA, want: "Let's Encrypt"},
+		{name: "staging", ca: certmagic.LetsEncryptStagingCA, want: "Let's Encrypt (staging)"},
+		{name: "ZeroSSL", ca: certmagic.ZeroSSLProductionCA, want: "ZeroSSL"},
+		{name: "Google Trust Services", ca: certmagic.GoogleTrustProductionCA, want: "Google Trust Services"},
+		{name: "Google Trust Services staging", ca: certmagic.GoogleTrustStagingCA, want: "Google Trust Services (staging)"},
+		{name: "other", ca: "https://ca.example/directory", want: "https://ca.example/directory"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, formatIssuer(api.CertificateIssuerData{
+				ACME: &api.ACMEIssuerData{CA: tt.ca},
+			}))
+		})
+	}
+
+	assert.Equal(t, "unknown", formatIssuer(api.CertificateIssuerData{}))
+}
+
+func TestFormatExpiry_UTC(t *testing.T) {
+	now := time.Now().UTC()
+	brisbane := time.FixedZone("AEST", 10*60*60)
+	expires := time.Date(now.Year(), now.Month(), now.Day()+3, 3, 0, 0, 0, brisbane)
+	assert.True(t, strings.HasPrefix(formatExpiry(expires), expires.UTC().Format(time.DateOnly)+" ("))
+	assert.NotEqual(t, expires.Format(time.DateOnly), expires.UTC().Format(time.DateOnly))
+	expired := now.Add(-2 * time.Hour)
+	assert.True(t, strings.HasPrefix(formatExpiry(expired), expired.UTC().Format(time.DateOnly)+" (expired "))
+	assert.Contains(t, formatExpiry(expired), " ago)")
+}
+
+func testIssuedCertificate(name, raw string, expires time.Time, ca string) api.IssuedCertificate {
+	cert := api.IssuedCertificate{
+		SAN:   name,
+		Chain: []*x509.Certificate{{Raw: []byte(raw), NotAfter: expires}},
+	}
+	if ca != "" {
+		cert.IssuerData.ACME = &api.ACMEIssuerData{CA: ca}
+	}
+	return cert
+}
+
+func testFingerprint(raw string) string {
+	sum := sha256.Sum256([]byte(raw))
+	return hex.EncodeToString(sum[:])
+}

cmd/uc/caddy/cert/root.go

@@ -0,0 +1,12 @@
+package cert
+
+import "github.com/spf13/cobra"
+
+func NewRootCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "cert",
+		Short: "Inspect certificates in cluster storage for Caddy.",
+	}
+	cmd.AddCommand(NewListCommand())
+	return cmd
+}

cmd/uc/caddy/config.go

@@ -9,6 +9,7 @@ import (
 	"github.com/psviderski/uncloud/internal/cli"
 	"github.com/psviderski/uncloud/internal/cli/completion"
 	"github.com/psviderski/uncloud/internal/cli/tui"
+	"github.com/psviderski/uncloud/pkg/client"
 	"github.com/spf13/cobra"
 )
 
@@ -47,12 +48,7 @@ func runConfig(ctx context.Context, uncli *cli.CLI, opts configOptions) error {
 	}
 	defer clusterClient.Close()
 
-	if opts.machine != "" {
+	config, err := clusterClient.Caddy.Config(ctx, client.CaddyConfigOptions{Machine: opts.machine})
-		// If a specific machine is requested, use it to get the Caddy configuration.
-		ctx = clusterClient.ProxySingleMachineContext(ctx, opts.machine)
-	}
-
-	config, err := clusterClient.Caddy.GetConfig(ctx, nil)
 	if err != nil {
 		return fmt.Errorf("get Caddy config: %w", err)
 	}

cmd/uc/caddy/deploy.go

@@ -110,7 +110,9 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 	placement := api.Placement{
 		Machines: cli.ExpandCommaSeparatedValues(opts.machines),
 	}
-	d, err := clusterClient.NewCaddyDeployment(opts.image, caddyfile, placement)
+	d, err := clusterClient.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{
+		Image: opts.image, Config: caddyfile, Placement: placement,
+	})
 	if err != nil {
 		return fmt.Errorf("create caddy deployment: %w", err)
 	}

cmd/uc/caddy/root.go

@@ -1,6 +1,7 @@
 package caddy
 
 import (
+	"github.com/psviderski/uncloud/cmd/uc/caddy/cert"
 	"github.com/spf13/cobra"
 )
 
@@ -10,6 +11,7 @@ func NewRootCommand() *cobra.Command {
 		Short: "Manage Caddy reverse proxy service.",
 	}
 	cmd.AddCommand(
+		cert.NewRootCommand(),
 		NewConfigCommand(),
 		NewDeployCommand(),
 		NewLogsCommand(),

cmd/uc/dns/set.go

@@ -9,8 +9,10 @@ import (
 	"github.com/psviderski/uncloud/api/pb"
 	"github.com/psviderski/uncloud/internal/cli"
 	"github.com/psviderski/uncloud/internal/cli/tui"
+	"github.com/psviderski/uncloud/internal/grpcversion"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 )
 
@@ -49,6 +51,7 @@ func set(ctx context.Context, uncli *cli.CLI, name string) error {
 	}
 	defer clusterClient.Close()
 
+	ctx = metadata.AppendToOutgoingContext(ctx, grpcversion.MetadataKeyMinServerVersion, "0.21.0")
 	_, err = clusterClient.SetDomain(ctx, &pb.SetDomainRequest{Name: name})
 	if err != nil {
 		if status.Convert(err).Code() == codes.AlreadyExists {

cmd/uc/machine/add.go

@@ -219,7 +219,7 @@ func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine,
 
 	fmt.Println()
 	fmt.Println("Preparing Caddy deployment...")
-	d, err := clusterClient.NewCaddyDeployment(caddyImage, "", api.Placement{})
+	d, err := clusterClient.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: caddyImage})
 	if err != nil {
 		return fmt.Errorf("create caddy deployment: %w", err)
 	}

cmd/uc/machine/init.go

@@ -16,7 +16,7 @@ import (
 	"github.com/psviderski/uncloud/internal/cli/tui"
 	"github.com/psviderski/uncloud/internal/machine/cluster"
 	"github.com/psviderski/uncloud/internal/machine/network"
-	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/psviderski/uncloud/pkg/client"
 	"github.com/spf13/cobra"
 )
 
@@ -221,17 +221,17 @@ func initCluster(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteM
 		initOpts.WireguardEndpoints = endpoints
 	}
 
-	client, err := uncli.InitCluster(ctx, initOpts)
+	clusterClient, err := uncli.InitCluster(ctx, initOpts)
 	if err != nil {
 		return err
 	}
-	defer client.Close()
+	defer clusterClient.Close()
 
 	// Since the cluster API needs a few moments to become ready after cluster initialisation,
 	// we keep the user informed during this wait. We wait here even if no Caddy or DNS is requested
 	// as the cluster needs to be ready so that commands such as 'uc machine ls' work immediately after init.
 	err = tui.RunSpinner(ctx, "Waiting for the cluster to be ready...", func(ctx context.Context) error {
-		return client.WaitClusterReady(ctx, 1*time.Minute)
+		return clusterClient.WaitClusterReady(ctx, 1*time.Minute)
 	})
 	if err != nil {
 		return fmt.Errorf("wait for cluster to be ready: %w", err)
@@ -245,7 +245,7 @@ func initCluster(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteM
 	fmt.Println()
 
 	if !opts.noDNS {
-		domain, err := client.ReserveDomain(ctx, &pb.ReserveDomainRequest{Endpoint: opts.dnsEndpoint})
+		domain, err := clusterClient.ReserveDomain(ctx, &pb.ReserveDomainRequest{Endpoint: opts.dnsEndpoint})
 		if err != nil {
 			return fmt.Errorf("reserve cluster domain in Uncloud DNS: %w", err)
 		}
@@ -253,7 +253,7 @@ func initCluster(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteM
 	}
 
 	if !opts.noCaddy {
-		d, err := client.NewCaddyDeployment("", "", api.Placement{})
+		d, err := clusterClient.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{})
 		if err != nil {
 			return fmt.Errorf("create caddy deployment: %w", err)
 		}
@@ -269,7 +269,7 @@ func initCluster(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteM
 		}
 
 		fmt.Println()
-		return caddy.UpdateDomainRecords(ctx, client, uncli.ProgressOut())
+		return caddy.UpdateDomainRecords(ctx, clusterClient, uncli.ProgressOut())
 	}
 
 	return nil

experiment/go.mod

@@ -1,6 +1,6 @@
 module github.com/psviderski/uncloud/experiment
 
-go 1.26
+go 1.26.0
 
 require (
 	github.com/dgraph-io/badger/v3 v3.2103.5
@@ -18,10 +18,10 @@ require (
 	github.com/psviderski/uncloud v0.0.0
 	github.com/siderolabs/discovery-api v0.1.4
 	github.com/siderolabs/discovery-client v0.1.9
-	go.uber.org/zap v1.27.0
+	go.uber.org/zap v1.28.0
-	golang.org/x/net v0.43.0
+	golang.org/x/net v0.59.0
 	golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173
-	google.golang.org/grpc v1.74.2
+	google.golang.org/grpc v1.81.0
 )
 
 require (
@@ -79,18 +79,18 @@ require (
 	github.com/ishidawataru/sctp v0.0.0-20230406120618-7ff4192f6ff2 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/josharian/native v1.1.0 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.19.1 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.4.0 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-libp2p v0.35.4 // indirect
 	github.com/libp2p/go-libp2p-pubsub v0.11.0 // indirect
 	github.com/libp2p/go-msgio v0.3.0 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mdlayher/genetlink v1.3.2 // indirect
 	github.com/mdlayher/netlink v1.7.2 // indirect
 	github.com/mdlayher/socket v0.5.1 // indirect
-	github.com/miekg/dns v1.1.65 // indirect
+	github.com/miekg/dns v1.1.73 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/cli v1.1.5 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
@@ -113,30 +113,30 @@ require (
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/siderolabs/gen v0.4.8 // indirect
-	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/sirupsen/logrus v1.9.4 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/cast v1.7.0 // indirect
 	github.com/vishvananda/netlink v1.3.1 // indirect
 	github.com/vishvananda/netns v0.0.5 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel v1.43.0 // indirect
-	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
-	go.opentelemetry.io/otel/trace v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.43.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
-	golang.org/x/crypto v0.41.0 // indirect
+	golang.org/x/crypto v0.57.0 // indirect
 	golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
-	golang.org/x/mod v0.27.0 // indirect
+	golang.org/x/mod v0.41.0 // indirect
-	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sync v0.23.0 // indirect
-	golang.org/x/sys v0.45.0 // indirect
+	golang.org/x/sys v0.48.0 // indirect
-	golang.org/x/text v0.28.0 // indirect
+	golang.org/x/text v0.42.0 // indirect
-	golang.org/x/time v0.11.0 // indirect
+	golang.org/x/time v0.16.0 // indirect
-	golang.org/x/tools v0.36.0 // indirect
+	golang.org/x/tools v0.50.0 // indirect
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
-	google.golang.org/protobuf v1.36.9 // indirect
+	google.golang.org/protobuf v1.36.12 // indirect
 	gotest.tools/v3 v3.5.2 // indirect
 	gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 // indirect
 	lukechampine.com/blake3 v1.3.0 // indirect

experiment/go.sum

@@ -302,8 +302,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
 github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
+github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0=
 github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk=
@@ -349,6 +351,7 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
@@ -368,6 +371,7 @@ github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKju
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
 github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc=
 github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck=
+github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ=
 github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=
 github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
 github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc=
@@ -529,6 +533,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
 github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
 github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
 github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
@@ -581,16 +586,20 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
 go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
+go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
 go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
 go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
+go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
 go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=
 go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY=
 go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis=
 go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4=
 go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
 go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
+go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc=
 go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
@@ -607,6 +616,7 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8
 go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -622,6 +632,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
 golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
+golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
 golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
@@ -635,6 +646,7 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
 golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
+golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -654,6 +666,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
 golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
+golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -666,6 +679,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
 golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -700,6 +714,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
 golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
@@ -711,8 +726,10 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
 golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
+golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
 golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
 golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
+golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -727,6 +744,7 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
 golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
+golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -745,6 +763,7 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
@@ -753,6 +772,7 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4=
 google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM=
+google.golang.org/grpc v1.81.0/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -764,6 +784,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
 google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/mattn/go-shellwords v1.0.12
+	github.com/mholt/acmez/v3 v3.1.6
 	github.com/miekg/dns v1.1.73
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/moby/term v0.5.2
@@ -178,7 +179,6 @@ require (
 	github.com/mdlayher/netlink v1.7.2 // indirect
 	github.com/mdlayher/socket v0.5.1 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
-	github.com/mholt/acmez/v3 v3.1.6 // indirect
 	github.com/miekg/pkcs11 v1.1.2 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect

internal/cli/cli.go

@@ -458,7 +458,7 @@ func (cli *CLI) AddMachine(ctx context.Context, opts AddMachineOptions) (_ *clie
 	}
 
 	// Snapshot the cluster store version so the new machine can catch up before participating.
-	var storeVersion map[string]uint64
+	var storeVersion api.StoreVersion
 	inspectResp, err = c.MachineClient.InspectMachine(ctx, &emptypb.Empty{})
 	if err != nil {
 		// TODO(lhf): remove Unimplemented check when v0.17.0 is released.

internal/corrosion/admin.go

@@ -13,6 +13,8 @@ import (
 	"time"
 )
 
+const adminCommandTimeout = 5 * time.Second
+
 // AdminClient is a client for the Corrosion admin API.
 type AdminClient struct {
 	sockPath string
@@ -32,10 +34,15 @@ type Response struct {
 // The channel will be closed after sending the last or error response. The caller must read from the channel until
 // it is closed.
 func (c *AdminClient) SendCommand(cmd []byte) (<-chan Response, error) {
-	conn, err := net.Dial("unix", c.sockPath)
+	// TODO: Accept contexts in admin methods so callers can cancel requests and set their deadlines.
+	conn, err := net.DialTimeout("unix", c.sockPath, adminCommandTimeout)
 	if err != nil {
 		return nil, fmt.Errorf("connect to admin socket: %w", err)
 	}
+	if err = conn.SetDeadline(time.Now().Add(adminCommandTimeout)); err != nil {
+		conn.Close()
+		return nil, fmt.Errorf("set admin connection deadline: %w", err)
+	}
 
 	if _, err = conn.Write(encodeFrame(cmd)); err != nil {
 		conn.Close()

internal/machine/caddystorage/certificate.go

@@ -0,0 +1,79 @@
+package caddystorage
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"strings"
+
+	"github.com/caddyserver/certmagic"
+	"github.com/psviderski/uncloud/api/pb"
+	"github.com/psviderski/uncloud/internal/machine/store"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+// ListCertificates lists issued certificates from this machine's Caddy storage replica.
+// Unreadable entries and invalid resource metadata are skipped. Chains and issuer data are returned as stored.
+func (s *Server) ListCertificates(ctx context.Context, _ *emptypb.Empty) (*pb.ListCertificatesResponse, error) {
+	records, err := s.store.List(ctx, "certificates/", store.KeyspaceListOptions{KeysOnly: true})
+	if err != nil {
+		if ctx.Err() != nil {
+			return nil, status.FromContextError(ctx.Err()).Err()
+		}
+		return nil, status.Errorf(codes.Internal, "list Caddy certificates: %v", err)
+	}
+
+	resp := &pb.ListCertificatesResponse{}
+	for _, record := range records {
+		if err = ctx.Err(); err != nil {
+			return nil, status.FromContextError(err).Err()
+		}
+		if !isCertificateKey(record.Key) {
+			continue
+		}
+		cert, err := s.loadIssuedCertificate(ctx, record.Key)
+		if err != nil {
+			slog.Warn("Failed to load issued certificate from Caddy storage.", "key", record.Key, "err", err)
+			continue
+		}
+		resp.Certificates = append(resp.Certificates, cert)
+	}
+
+	return resp, nil
+}
+
+// isCertificateKey recognizes CertMagic's certificates/<issuer>/<name>/<name>.crt layout.
+func isCertificateKey(key string) bool {
+	parts := strings.Split(key, "/")
+	return len(parts) == 4 && parts[0] == "certificates" && parts[3] == parts[2]+".crt"
+}
+
+func (s *Server) loadIssuedCertificate(ctx context.Context, key string) (*pb.IssuedCertificate, error) {
+	chain, err := s.store.Get(ctx, key)
+	if err != nil {
+		return nil, err
+	}
+
+	metaKey := strings.TrimSuffix(key, ".crt") + ".json"
+	meta, err := s.store.Get(ctx, metaKey)
+	if err != nil {
+		return nil, err
+	}
+
+	var resource certmagic.CertificateResource
+	if err = json.Unmarshal(meta.Value, &resource); err != nil {
+		return nil, err
+	}
+	if len(resource.SANs) == 0 || strings.TrimSpace(resource.SANs[0]) == "" {
+		return nil, fmt.Errorf("certificate metadata has no SAN")
+	}
+
+	return &pb.IssuedCertificate{
+		San:        resource.SANs[0],
+		Chain:      chain.Value,
+		IssuerData: resource.IssuerData,
+	}, nil
+}

internal/machine/caddystorage/certificate_test.go

@@ -0,0 +1,233 @@
+package caddystorage
+
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"math/big"
+	"net"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/caddyserver/certmagic"
+	"github.com/psviderski/uncloud/internal/machine/store"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+func TestServerListCertificates(t *testing.T) {
+	for _, sans := range [][]string{
+		{"app.example.com"}, {"*.example.com"}, {"192.0.2.1"}, {"www.example.com", "example.com"},
+	} {
+		t.Run(sans[0], func(t *testing.T) {
+			bundle, _, _ := certificateFixture(t, &x509.Certificate{
+				DNSNames:    []string{"example.com", "www.example.com"},
+				IPAddresses: []net.IP{net.ParseIP("192.0.2.1")},
+				NotAfter:    time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC),
+			})
+			key := certmagic.StorageKeys.SiteCert("local", sans[0])
+			storage := newCertificateStorage(t)
+			raw := json.RawMessage(`{"unknown_issuer_field":true}`)
+			storage.add(t, key, bundle, sans, raw)
+			storage.keys = append(storage.keys, "certificates", "certificates/local", "certificates/local/app.crt",
+				"certificates/local/app/wrong.crt", "certificates/local/app/app.crt/extra.crt",
+				"pki/authorities/local/root.crt", "ocsp/certificate")
+
+			resp, err := NewServer(storage).ListCertificates(t.Context(), &emptypb.Empty{})
+			require.NoError(t, err)
+			require.Len(t, resp.Certificates, 1)
+			cert := resp.Certificates[0]
+			assert.Equal(t, sans[0], cert.San)
+			assert.Equal(t, bundle, cert.Chain, "PEM must be returned unchanged")
+			assert.Equal(t, []byte(raw), cert.IssuerData)
+			assert.Equal(t, []string{key, strings.TrimSuffix(key, ".crt") + ".json"}, storage.loads)
+		})
+	}
+}
+
+func TestServerListCertificates_PreservesIssuerData(t *testing.T) {
+	bundle, _, _ := certificateFixture(t, &x509.Certificate{})
+	for _, raw := range []string{
+		"", "null", `{"url":"https://ca/cert/1","ca":"https://ca/directory"}`,
+		`{"url":"https://ca/cert/1","ca":"https://ca/directory","renewal_info":{"_selectedTime":"invalid"}}`,
+		`"provider-record"`, "[1,2,3]",
+	} {
+		t.Run(raw, func(t *testing.T) {
+			storage := newCertificateStorage(t)
+			storage.add(t, "certificates/custom/app/app.crt", bundle, []string{"app"}, json.RawMessage(raw))
+
+			resp, err := NewServer(storage).ListCertificates(t.Context(), &emptypb.Empty{})
+			require.NoError(t, err)
+			require.Len(t, resp.Certificates, 1)
+			assert.Equal(t, raw, string(resp.Certificates[0].IssuerData))
+		})
+	}
+}
+
+func TestServerListCertificates_OrderingAndDuplicates(t *testing.T) {
+	bundle, _, _ := certificateFixture(t, &x509.Certificate{})
+	storage := newCertificateStorage(t)
+	storage.add(t, certmagic.StorageKeys.SiteCert("z-issuer", "a.example.com"), bundle, []string{"a.example.com"}, nil)
+	storage.add(t, certmagic.StorageKeys.SiteCert("a-issuer", "z.example.com"), bundle, []string{"z.example.com"}, nil)
+	storage.add(t, certmagic.StorageKeys.SiteCert("b-issuer", "z.example.com"), bundle, []string{"z.example.com"}, nil)
+
+	resp, err := NewServer(storage).ListCertificates(t.Context(), &emptypb.Empty{})
+	require.NoError(t, err)
+	require.Len(t, resp.Certificates, 3)
+	certs := resp.Certificates
+	assert.Equal(t, []string{"z.example.com", "z.example.com", "a.example.com"},
+		[]string{certs[0].San, certs[1].San, certs[2].San})
+	for _, cert := range certs {
+		assert.Equal(t, bundle, cert.Chain)
+	}
+}
+
+func TestServerListCertificates_SkipsUnreadableEntries(t *testing.T) {
+	bundle, _, _ := certificateFixture(t, &x509.Certificate{})
+	const badKey = "certificates/local/bad/bad.crt"
+	const metaKey = "certificates/local/bad/bad.json"
+	for _, tt := range []struct {
+		name   string
+		change func(*certificateStorageStub)
+	}{
+		{"missing chain", func(s *certificateStorageStub) { delete(s.values, badKey) }},
+		{"chain read failure", func(s *certificateStorageStub) { s.loadErrors[badKey] = errors.New("offline") }},
+		{"missing metadata", func(s *certificateStorageStub) { delete(s.values, metaKey) }},
+		{"metadata read failure", func(s *certificateStorageStub) { s.loadErrors[metaKey] = errors.New("offline") }},
+		{"invalid JSON", func(s *certificateStorageStub) { s.values[metaKey] = []byte("{") }},
+		{"missing SAN", func(s *certificateStorageStub) { s.values[metaKey] = []byte(`{}`) }},
+		{"empty first SAN", func(s *certificateStorageStub) { s.values[metaKey] = []byte(`{"sans":["","app"]}`) }},
+		{"blank first SAN", func(s *certificateStorageStub) { s.values[metaKey] = []byte(`{"sans":[" "]}`) }},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			storage := newCertificateStorage(t)
+			storage.add(t, badKey, bundle, []string{"bad"}, nil)
+			storage.add(t, "certificates/local/good/good.crt", bundle, []string{"good"}, nil)
+			tt.change(storage)
+
+			resp, err := NewServer(storage).ListCertificates(t.Context(), &emptypb.Empty{})
+			require.NoError(t, err)
+			require.Len(t, resp.Certificates, 1)
+			assert.Equal(t, "good", resp.Certificates[0].San)
+		})
+	}
+
+	t.Run("all entries unreadable", func(t *testing.T) {
+		storage := newCertificateStorage(t)
+		storage.keys = []string{badKey}
+		resp, err := NewServer(storage).ListCertificates(t.Context(), &emptypb.Empty{})
+		require.NoError(t, err)
+		assert.Empty(t, resp.Certificates)
+	})
+}
+
+func TestServerListCertificates_ListErrors(t *testing.T) {
+	for _, fail := range []bool{false, true} {
+		storage := newCertificateStorage(t)
+		if fail {
+			storage.listError = errors.New("offline")
+		}
+
+		resp, err := NewServer(storage).ListCertificates(t.Context(), &emptypb.Empty{})
+		assert.Empty(t, storage.loads)
+		if fail {
+			assert.Nil(t, resp)
+			assert.Equal(t, codes.Internal, status.Code(err))
+		} else {
+			require.NoError(t, err)
+			assert.Empty(t, resp.Certificates)
+		}
+	}
+}
+
+// Only reads are implemented. Unexpected writes panic through the nil embedded Keyspace.
+type certificateStorageStub struct {
+	*store.Keyspace
+	t          *testing.T
+	keys       []string
+	values     map[string][]byte
+	loads      []string
+	listCalls  int
+	listError  error
+	loadErrors map[string]error
+	onLoad     func(string)
+}
+
+func newCertificateStorage(t *testing.T) *certificateStorageStub {
+	return &certificateStorageStub{t: t, values: make(map[string][]byte), loadErrors: make(map[string]error)}
+}
+
+func (s *certificateStorageStub) add(t *testing.T, key string, bundle []byte, sans []string, issuerData json.RawMessage) {
+	t.Helper()
+	metaKey := strings.TrimSuffix(key, ".crt") + ".json"
+	keyKey := strings.TrimSuffix(key, ".crt") + ".key"
+	meta, err := json.Marshal(certmagic.CertificateResource{SANs: sans, IssuerData: issuerData})
+	require.NoError(t, err)
+	s.keys = append(s.keys, key, metaKey, keyKey)
+	s.values[key], s.values[metaKey], s.values[keyKey] = bundle, meta, []byte("must not read private keys")
+}
+
+func (s *certificateStorageStub) List(ctx context.Context, prefix string, opts store.KeyspaceListOptions) ([]store.Record, error) {
+	s.listCalls++
+	assert.Equal(s.t, "certificates/", prefix)
+	assert.True(s.t, opts.KeysOnly, "listing must not fetch private key values")
+	var records []store.Record
+	for _, key := range slices.Sorted(slices.Values(s.keys)) {
+		records = append(records, store.Record{Key: key})
+	}
+	return records, s.listError
+}
+
+func (s *certificateStorageStub) Get(ctx context.Context, key string) (store.Record, error) {
+	require.False(s.t, strings.HasSuffix(key, ".key"), "private keys must never be loaded")
+	s.loads = append(s.loads, key)
+	if s.onLoad != nil {
+		s.onLoad(key)
+	}
+	if err := s.loadErrors[key]; err != nil {
+		return store.Record{}, err
+	}
+	value, ok := s.values[key]
+	if !ok {
+		return store.Record{}, store.ErrKeyNotFound
+	}
+	return store.Record{Key: key, Value: value}, nil
+}
+
+func certificateFixture(t *testing.T, template *x509.Certificate) ([]byte, *x509.Certificate, *x509.Certificate) {
+	t.Helper()
+	publicKey, key, err := ed25519.GenerateKey(rand.Reader)
+	require.NoError(t, err)
+	root := &x509.Certificate{
+		SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Test CA"}, IsCA: true,
+		BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign,
+		NotBefore: time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC),
+		NotAfter:  time.Date(2040, 1, 1, 0, 0, 0, 0, time.UTC),
+	}
+	rootDER, err := x509.CreateCertificate(rand.Reader, root, root, publicKey, key)
+	require.NoError(t, err)
+	root, err = x509.ParseCertificate(rootDER)
+	require.NoError(t, err)
+	template.SerialNumber = big.NewInt(2)
+	template.NotBefore = root.NotBefore
+	if template.NotAfter.IsZero() {
+		template.NotAfter = root.NotAfter
+	}
+	leafDER, err := x509.CreateCertificate(rand.Reader, template, root, publicKey, key)
+	require.NoError(t, err)
+	leaf, err := x509.ParseCertificate(leafDER)
+	require.NoError(t, err)
+	bundle := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}),
+		pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: rootDER})...)
+	return bundle, leaf, root
+}

internal/machine/caddystorage/server.go

@@ -25,10 +25,17 @@ const Namespace = "caddy_storage"
 // Server implements the machine-local CaddyStorage gRPC service.
 type Server struct {
 	pb.UnimplementedCaddyStorageServer
-	store *store.Keyspace
+	store keyspace
 }
 
-func NewServer(store *store.Keyspace) *Server {
+type keyspace interface {
+	Get(context.Context, string) (store.Record, error)
+	Put(context.Context, string, []byte) error
+	Delete(context.Context, string, store.KeyspaceDeleteOptions) error
+	List(context.Context, string, store.KeyspaceListOptions) ([]store.Record, error)
+}
+
+func NewServer(store keyspace) *Server {
 	return &Server{store: store}
 }
 

internal/machine/machine.go

@@ -262,11 +262,11 @@ func NewMachine(config *Config) (*Machine, error) {
 	if err != nil {
 		return nil, fmt.Errorf("create corrosion API client: %w", err)
 	}
-	corroStore := store.New(corro)
 	corroAdmin, err := corrosion.NewAdminClient(config.CorrosionAdminSockPath)
 	if err != nil {
 		return nil, fmt.Errorf("create corrosion admin client: %w", err)
 	}
+	corroStore := store.New(corro, corroAdmin)
 
 	initialised := make(chan struct{})
 	clusterReady := make(chan struct{})

internal/machine/state.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/psviderski/uncloud/internal/machine/network"
 	"github.com/psviderski/uncloud/internal/secret"
+	"github.com/psviderski/uncloud/pkg/api"
 )
 
 const (
@@ -31,7 +32,7 @@ type State struct {
 	// MinStoreVersion is the cluster store version this machine must reach before participating.
 	// Per-actor vector (Corrosion actor UUID → max processed db_version) captured from an existing
 	// member at join time. Cleared once reached.
-	MinStoreVersion map[string]uint64 `json:",omitempty"`
+	MinStoreVersion api.StoreVersion `json:",omitempty"`
 	// CorrosionAPIToken authenticates requests to the local Corrosion API.
 	CorrosionAPIToken secret.Secret `json:",omitempty"`
 

internal/machine/store/replication.go

@@ -8,14 +8,16 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/psviderski/uncloud/internal/corrosion"
+	"github.com/psviderski/uncloud/pkg/api"
 )
 
 // ErrInvalidStoreVersion indicates an invalid actor UUID in a store version vector.
 var ErrInvalidStoreVersion = errors.New("invalid store version")
 
 // WaitForVersion waits until the local store has reached each actor's minimum version in minVersion, with no known
-// missing or pending transactions through those versions. Corrosion may satisfy a version by applying its surviving
+// missing or pending transactions through those versions from active members. Corrosion may satisfy a version
-// changes or by marking it complete because its changes have been superseded.
+// by applying its surviving changes or by marking it complete because its changes have been superseded.
 //
 // Waiting normally makes the captured data available locally. However, if another write replaces some of that data
 // before it arrives, Corrosion can complete the older version without transferring the replaced data. If the
@@ -27,7 +29,7 @@ var ErrInvalidStoreVersion = errors.New("invalid store version")
 //
 // The method observes native replication without initiating synchronisation.
 // An empty minVersion requires no replication. The context controls cancellation and the deadline.
-func (s *Store) WaitForVersion(ctx context.Context, minVersion map[string]uint64) error {
+func (s *Store) WaitForVersion(ctx context.Context, minVersion api.StoreVersion) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
@@ -52,28 +54,64 @@ func (s *Store) WaitForVersion(ctx context.Context, minVersion map[string]uint64
 		return nil
 	}
 
-	// Check all bookkeeping in one SQLite snapshot. Raw buffered rows can remain after application,
+	queryPrefix := `WITH min_version(actor_id, version) AS (VALUES ` + strings.Join(placeholders, ", ") + `)
-	// but sequence bookkeeping is removed in the same transaction that applies or clears a version.
-	query := `WITH min_version(actor_id, version) AS (VALUES ` + strings.Join(placeholders, ", ") + `)
 		SELECT NOT EXISTS (
 			SELECT 1 FROM min_version
 			LEFT JOIN crsql_db_versions AS current ON current.site_id = min_version.actor_id
-			WHERE COALESCE(current.db_version, 0) < min_version.version
+			WHERE COALESCE(current.db_version, 0) < min_version.version`
-				OR EXISTS (
+
-					SELECT 1 FROM __corro_bookkeeping_gaps
+	versionsReached := false
-					WHERE actor_id = min_version.actor_id AND start <= min_version.version
-				)
-				OR EXISTS (
-					SELECT 1 FROM __corro_seq_bookkeeping
-					WHERE site_id = min_version.actor_id AND db_version <= min_version.version
-				)
-		)`
 
 	ticker := time.NewTicker(100 * time.Millisecond)
 	defer ticker.Stop()
 
 	for {
-		rows, err := s.corro.QueryContext(ctx, query, args...)
+		query := queryPrefix
+		queryArgs := args
+		// Check for gaps and pending transactions only after the requested versions are reached.
+		if versionsReached {
+			states, err := s.corroAdmin.ClusterMembershipStates(true)
+			if err != nil {
+				return fmt.Errorf("get cluster membership for store replication: %w", err)
+			}
+
+			activePlaceholders := make([]string, 0, len(states))
+			queryArgs = make([]any, len(args), len(args)+len(states))
+			copy(queryArgs, args)
+			for _, state := range states {
+				if state.State != corrosion.MembershipStateAlive && state.State != corrosion.MembershipStateSuspect {
+					continue
+				}
+				actorID, err := uuid.Parse(state.ID)
+				if err != nil {
+					return fmt.Errorf("parse cluster member actor '%s': %w", state.ID, err)
+				}
+				activePlaceholders = append(activePlaceholders, "?")
+				queryArgs = append(queryArgs, [16]byte(actorID))
+			}
+			if len(activePlaceholders) > 0 {
+				// Check active members' gaps and pending sequences in the same query for simplicity.
+				// Buffered rows can remain in __corro_buffered_changes after application, but sequence bookkeeping
+				// is removed when a version completes.
+				query += `
+				OR (
+					min_version.actor_id IN (` + strings.Join(activePlaceholders, ", ") + `)
+					AND (
+						EXISTS (
+							SELECT 1 FROM __corro_bookkeeping_gaps
+							WHERE actor_id = min_version.actor_id AND start <= min_version.version
+						)
+						OR EXISTS (
+							SELECT 1 FROM __corro_seq_bookkeeping
+							WHERE site_id = min_version.actor_id AND db_version <= min_version.version
+						)
+					)
+				)`
+			}
+		}
+		query += ")"
+
+		rows, err := s.corro.QueryContext(ctx, query, queryArgs...)
 		if err != nil {
 			return fmt.Errorf("check store replication: %w", err)
 		}
@@ -83,6 +121,7 @@ func (s *Store) WaitForVersion(ctx context.Context, minVersion map[string]uint64
 			}
 			return errors.New("check store replication: store version query returned no rows")
 		}
+
 		var reached int
 		if err = rows.Scan(&reached); err != nil {
 			rows.Close()
@@ -94,7 +133,11 @@ func (s *Store) WaitForVersion(ctx context.Context, minVersion map[string]uint64
 			return fmt.Errorf("check store replication: %w", err)
 		}
 		if reached == 1 {
-			return ctx.Err()
+			if versionsReached {
+				return nil
+			}
+			versionsReached = true
+			continue
 		}
 
 		select {

internal/machine/store/store.go

@@ -10,6 +10,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/psviderski/uncloud/api/pb"
 	"github.com/psviderski/uncloud/internal/corrosion"
+	"github.com/psviderski/uncloud/pkg/api"
 	"google.golang.org/protobuf/encoding/protojson"
 )
 
@@ -23,11 +24,12 @@ var (
 
 // Store is a cluster store backed by a distributed Corrosion database.
 type Store struct {
-	corro *corrosion.APIClient
+	corro      *corrosion.APIClient
+	corroAdmin *corrosion.AdminClient
 }
 
-func New(corro *corrosion.APIClient) *Store {
+func New(corro *corrosion.APIClient, corroAdmin *corrosion.AdminClient) *Store {
-	return &Store{corro: corro}
+	return &Store{corro: corro, corroAdmin: corroAdmin}
 }
 
 // Get retrieves an unnamespaced legacy value.
@@ -81,14 +83,14 @@ func (s *Store) Delete(ctx context.Context, key string) error {
 // limitations documented there.
 //
 // Capturing a vector does not wait for replication or prevent further writes.
-func (s *Store) Version(ctx context.Context) (map[string]uint64, error) {
+func (s *Store) Version(ctx context.Context) (api.StoreVersion, error) {
 	rows, err := s.corro.QueryContext(ctx, "SELECT site_id, db_version FROM crsql_db_versions")
 	if err != nil {
 		return nil, fmt.Errorf("query crsql_db_versions: %w", err)
 	}
 	defer rows.Close()
 
-	versions := make(map[string]uint64)
+	versions := make(api.StoreVersion)
 	for rows.Next() {
 		var (
 			siteID  []byte

pkg/api/caddy.go

@@ -1,6 +1,18 @@
 package api
 
-import "strings"
+import (
+	"strings"
+	"time"
+)
+
+// CaddyConfig is the saved Caddy configuration on a machine.
+type CaddyConfig struct {
+	Caddyfile string
+	// ModifiedAt is zero when the server does not supply a modification timestamp.
+	ModifiedAt time.Time
+	// LastReconciliationError describes the latest unsuccessful configuration reconciliation.
+	LastReconciliationError string
+}
 
 // CaddySpec is the Caddy reverse proxy configuration for a service.
 type CaddySpec struct {

pkg/api/caddy_certificate.go

@@ -0,0 +1,111 @@
+package api
+
+import (
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"strings"
+
+	"github.com/mholt/acmez/v3/acme"
+	"github.com/psviderski/uncloud/api/pb"
+)
+
+// IssuedCertificate describes a certificate in Caddy's managed certificate storage.
+// It does not indicate whether Caddy currently serves the certificate or whether it is trusted.
+type IssuedCertificate struct {
+	// SAN is the Subject Alternative Name (SAN) of the certificate.
+	// Caddy doesn't issue certificates with multiple SANs.
+	SAN string
+	// Chain contains the parsed certificates in stored order, with the leaf first.
+	Chain []*x509.Certificate
+	// IssuerData is extra information associated with the certificate, usually provided by the issuer implementation.
+	IssuerData CertificateIssuerData
+}
+
+// Fingerprint returns the SHA-256 fingerprint of the leaf certificate.
+func (c IssuedCertificate) Fingerprint() [sha256.Size]byte {
+	return sha256.Sum256(c.Chain[0].Raw)
+}
+
+// CertificateIssuerData preserves issuer-specific metadata and provides a best-effort typed view of ACME records.
+type CertificateIssuerData struct {
+	// Raw is the original issuer_data JSON, including unrecognized formats and fields.
+	Raw json.RawMessage
+	// ACME is populated when the metadata can be decoded as an ACME record with CA and certificate URLs.
+	// It is nil for absent, unrecognized, or malformed metadata.
+	ACME *ACMEIssuerData
+}
+
+// ACMEIssuerData identifies the ACME resources used to issue a certificate.
+type ACMEIssuerData struct {
+	// URL is the certificate resource URL as provisioned by the ACME server.
+	URL string
+	// CA is the directory URL of the ACME CA that issued this certificate.
+	CA string
+	// Account is the URL of the account that obtained the certificate.
+	Account string
+	// RenewalInfo is the stored renewal guidance, not a guarantee of when renewal will run.
+	RenewalInfo *acme.RenewalInfo
+}
+
+// IssuedCertificateFromProto parses a stored certificate chain without verifying trust or expiry.
+// Issuer metadata is decoded as ACME on a best-effort basis and is always preserved in its raw form.
+func IssuedCertificateFromProto(p *pb.IssuedCertificate) (IssuedCertificate, error) {
+	if p == nil || strings.TrimSpace(p.San) == "" {
+		return IssuedCertificate{}, fmt.Errorf("invalid certificate: missing SAN")
+	}
+	chain, err := parseCertificateChain(p.Chain)
+	if err != nil {
+		return IssuedCertificate{}, fmt.Errorf("parse certificate '%s': %w", p.San, err)
+	}
+	return IssuedCertificate{
+		SAN:        p.San,
+		Chain:      chain,
+		IssuerData: parseCertificateIssuerData(p.IssuerData),
+	}, nil
+}
+
+func parseCertificateChain(data []byte) ([]*x509.Certificate, error) {
+	var chain []*x509.Certificate
+	for len(data) > 0 {
+		var block *pem.Block
+		block, data = pem.Decode(data)
+		if block == nil {
+			break
+		}
+		if block.Type != "CERTIFICATE" {
+			return nil, fmt.Errorf("unexpected PEM block type '%s'", block.Type)
+		}
+
+		cert, err := x509.ParseCertificate(block.Bytes)
+		if err != nil {
+			return nil, fmt.Errorf("parse X.509 certificate: %w", err)
+		}
+		chain = append(chain, cert)
+	}
+
+	if len(chain) == 0 {
+		return nil, fmt.Errorf("empty certificate chain")
+	}
+
+	return chain, nil
+}
+
+func parseCertificateIssuerData(raw json.RawMessage) CertificateIssuerData {
+	data := CertificateIssuerData{Raw: raw}
+	// Recognize ACME by its resource URLs and decodable metadata.
+	// Unknown formats and malformed records remain raw-only.
+	var cert acme.Certificate
+	if err := json.Unmarshal(raw, &cert); err != nil || cert.CA == "" || cert.URL == "" {
+		return data
+	}
+	data.ACME = &ACMEIssuerData{
+		URL:         cert.URL,
+		CA:          cert.CA,
+		Account:     cert.Account,
+		RenewalInfo: cert.RenewalInfo,
+	}
+	return data
+}

pkg/api/caddy_certificate_test.go

@@ -0,0 +1,182 @@
+package api
+
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/pem"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/mholt/acmez/v3/acme"
+	"github.com/psviderski/uncloud/api/pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIssuedCertificateFromProto(t *testing.T) {
+	leaf := testCertificatePEM(t, 1)
+	issuer := testCertificatePEM(t, 2)
+	for _, tt := range []struct {
+		name    string
+		chain   []byte
+		serials []int64
+	}{
+		{name: "single certificate", chain: leaf, serials: []int64{1}},
+		{name: "certificate chain", chain: append(append([]byte(nil), leaf...), issuer...), serials: []int64{1, 2}},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			cert, err := IssuedCertificateFromProto(&pb.IssuedCertificate{
+				San: "app.example.com", Chain: tt.chain,
+			})
+			require.NoError(t, err)
+			assert.Equal(t, "app.example.com", cert.SAN)
+			require.Len(t, cert.Chain, len(tt.serials))
+
+			for i, serial := range tt.serials {
+				assert.Equal(t, big.NewInt(serial), cert.Chain[i].SerialNumber, "chain order must be preserved")
+			}
+			assert.Equal(t, []string{"app.example.com"}, cert.Chain[0].DNSNames)
+			// Expired, self-signed certificates are parsed without verifying trust or validity.
+			assert.Equal(t, time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), cert.Chain[0].NotAfter)
+			assert.Empty(t, cert.IssuerData)
+		})
+	}
+}
+
+func TestIssuedCertificateFingerprint(t *testing.T) {
+	cert := IssuedCertificate{Chain: []*x509.Certificate{
+		{Raw: []byte("abc")},
+		{Raw: []byte("issuer")},
+	}}
+	want, err := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
+	require.NoError(t, err)
+	fingerprint := cert.Fingerprint()
+	assert.Equal(t, want, fingerprint[:])
+}
+
+func TestIssuedCertificateFromProto_InvalidCertificate(t *testing.T) {
+	valid := testCertificatePEM(t, 1)
+	invalidDER := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte("invalid")})
+	for _, tt := range []struct {
+		name    string
+		input   *pb.IssuedCertificate
+		wantErr string
+	}{
+		{name: "nil input", wantErr: "missing SAN"},
+		{name: "missing SAN", input: &pb.IssuedCertificate{Chain: valid}, wantErr: "missing SAN"},
+		{name: "blank SAN", input: &pb.IssuedCertificate{San: " \t", Chain: valid}, wantErr: "missing SAN"},
+		{name: "empty chain", input: &pb.IssuedCertificate{San: "app.example.com"}, wantErr: "empty certificate chain"},
+		{
+			name: "invalid PEM", input: &pb.IssuedCertificate{San: "app.example.com", Chain: []byte("not PEM")},
+			wantErr: "empty certificate chain",
+		},
+		{
+			name: "wrong PEM block type", input: &pb.IssuedCertificate{
+				San:   "app.example.com",
+				Chain: pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: []byte("key")}),
+			},
+			wantErr: "unexpected PEM block type",
+		},
+		{
+			name: "invalid DER", input: &pb.IssuedCertificate{San: "app.example.com", Chain: invalidDER},
+			wantErr: "parse X.509 certificate",
+		},
+		{name: "invalid certificate after valid leaf", input: &pb.IssuedCertificate{
+			San:   "app.example.com",
+			Chain: append(append([]byte(nil), valid...), invalidDER...),
+		}, wantErr: "parse X.509 certificate"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			cert, err := IssuedCertificateFromProto(tt.input)
+			require.ErrorContains(t, err, tt.wantErr)
+			assert.Empty(t, cert)
+		})
+	}
+}
+
+func TestIssuedCertificateFromProto_IssuerData(t *testing.T) {
+	chain := testCertificatePEM(t, 1)
+	retryAfter := time.Date(2029, 12, 1, 0, 0, 0, 0, time.UTC)
+	renewalInfo := &acme.RenewalInfo{
+		ExplanationURL:   "https://ca.example/why",
+		UniqueIdentifier: "aki.serial",
+		RetryAfter:       &retryAfter,
+		SelectedTime:     time.Date(2030, 1, 2, 0, 0, 0, 0, time.UTC),
+	}
+	renewalInfo.SuggestedWindow.Start = time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)
+	renewalInfo.SuggestedWindow.End = time.Date(2030, 1, 3, 0, 0, 0, 0, time.UTC)
+
+	for _, tt := range []struct {
+		name string
+		raw  string
+		want *ACMEIssuerData
+	}{
+		{
+			name: "ACME",
+			raw:  `{"url":"https://ca.example/cert/1","ca":"https://ca.example/directory","account":"https://ca.example/acct/1","future_field":true}`,
+			want: &ACMEIssuerData{
+				URL: "https://ca.example/cert/1", CA: "https://ca.example/directory", Account: "https://ca.example/acct/1",
+			},
+		},
+		{
+			name: "ACME with renewal information",
+			raw: `{
+				"url": "https://ca.example/cert/1",
+				"ca": "https://ca.example/directory",
+				"account": "https://ca.example/acct/1",
+				"renewal_info": {
+					"suggestedWindow": {"start": "2030-01-01T00:00:00Z", "end": "2030-01-03T00:00:00Z"},
+					"explanationURL": "https://ca.example/why",
+					"_uniqueIdentifier": "aki.serial",
+					"_retryAfter": "2029-12-01T00:00:00Z",
+					"_selectedTime": "2030-01-02T00:00:00Z"
+				}
+			}`,
+			want: &ACMEIssuerData{
+				URL: "https://ca.example/cert/1", CA: "https://ca.example/directory",
+				Account: "https://ca.example/acct/1", RenewalInfo: renewalInfo,
+			},
+		},
+		{name: "absent"},
+		{name: "null", raw: "null"},
+		{name: "unknown issuer", raw: `{"id":"provider-id","status":"issued"}`},
+		{name: "non-object JSON", raw: `"provider-record"`},
+		{name: "missing CA", raw: `{"url":"https://ca.example/cert/1"}`},
+		{name: "missing certificate URL", raw: `{"ca":"https://ca.example/directory"}`},
+		{name: "invalid JSON", raw: "{"},
+		{name: "invalid field types", raw: `{"url":123,"ca":false}`},
+		{
+			name: "malformed renewal information",
+			raw:  `{"url":"https://ca.example/cert/1","ca":"https://ca.example/directory","renewal_info":{"_selectedTime":"invalid"}}`,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			cert, err := IssuedCertificateFromProto(&pb.IssuedCertificate{
+				San: "app.example.com", Chain: chain, IssuerData: []byte(tt.raw),
+			})
+			require.NoError(t, err, "issuer metadata must not prevent certificate parsing")
+			require.Len(t, cert.Chain, 1)
+			assert.Equal(t, tt.raw, string(cert.IssuerData.Raw), "preserve original JSON including unknown fields")
+			assert.Equal(t, tt.want, cert.IssuerData.ACME)
+		})
+	}
+}
+
+// testCertificatePEM creates an expired, self-signed certificate for parsing tests.
+func testCertificatePEM(t *testing.T, serial int64) []byte {
+	t.Helper()
+	public, private, err := ed25519.GenerateKey(rand.Reader)
+	require.NoError(t, err)
+	template := &x509.Certificate{
+		SerialNumber: big.NewInt(serial),
+		DNSNames:     []string{"app.example.com"},
+		NotBefore:    time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
+		NotAfter:     time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, public, private)
+	require.NoError(t, err)
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+}

pkg/api/store.go

@@ -0,0 +1,14 @@
+package api
+
+// StoreVersion records the highest processed database version for each Corrosion actor UUID.
+// A version can include changes that were superseded, so it does not guarantee that all data
+// through that version is available locally.
+type StoreVersion map[string]uint64
+
+// MergeMax keeps the highest observed version for each actor in other.
+// The receiver must be initialised before calling MergeMax.
+func (v StoreVersion) MergeMax(other StoreVersion) {
+	for actor, version := range other {
+		v[actor] = max(v[actor], version)
+	}
+}

pkg/client/caddy.go

@@ -1,15 +1,19 @@
 package client
 
 import (
+	"context"
 	"fmt"
 	"regexp"
+	"time"
 
 	"github.com/Masterminds/semver"
 	"github.com/distribution/reference"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/psviderski/uncloud/api/pb"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client/deploy"
+	"google.golang.org/protobuf/types/known/emptypb"
 )
 
 const (
@@ -20,12 +24,66 @@ const (
 
 var caddyImageTagRegex = regexp.MustCompile(`^2\.\d+\.\d+$`)
 
-// NewCaddyDeployment creates a new deployment for a Caddy reverse proxy service.
+// CaddyClient provides Caddy operations over its parent Client's connection.
+// The parent client owns the connection and must be used to close it.
+type CaddyClient struct {
+	// Storage provides low-level access to Caddy's cluster-backed storage.
+	Storage pb.CaddyStorageClient
+	grpc    pb.CaddyClient
+	client  *Client
+}
+
+// CaddyConfigOptions controls which machine's saved Caddy configuration is retrieved.
+type CaddyConfigOptions struct {
+	// Machine is the machine name or ID. If empty, the configuration is retrieved from the machine the client
+	// is connected to.
+	Machine string
+}
+
+// Config retrieves the saved Caddy configuration from the machine selected by opts.
+func (c *CaddyClient) Config(ctx context.Context, opts CaddyConfigOptions) (api.CaddyConfig, error) {
+	if opts.Machine != "" {
+		ctx = ProxySingleMachineContext(ctx, opts.Machine)
+	}
+
+	resp, err := c.grpc.GetConfig(ctx, &emptypb.Empty{})
+	if err != nil {
+		return api.CaddyConfig{}, fmt.Errorf("get Caddy config: %w", err)
+	}
+	config := api.CaddyConfig{
+		Caddyfile:               resp.Caddyfile,
+		LastReconciliationError: resp.LastReconciliationError,
+	}
+	if resp.ModifiedAt != nil {
+		if err := resp.ModifiedAt.CheckValid(); err != nil {
+			return api.CaddyConfig{}, fmt.Errorf("invalid Caddy config modification timestamp: %w", err)
+		}
+		config.ModifiedAt = resp.ModifiedAt.AsTime()
+	}
+
+	return config, nil
+}
+
+// CaddyDeploymentOptions configures a Caddy reverse proxy deployment.
+type CaddyDeploymentOptions struct {
+	// Image defaults to the latest stable 2.x.x official Caddy image.
+	// Custom images must include curl and start Caddy with /etc/caddy/Caddyfile.
+	Image string
+	// Config contains an optional global Caddyfile.
+	Config    string
+	Placement api.Placement
+}
+
+// NewDeployment creates a new deployment for a Caddy reverse proxy service.
 // The service is deployed in global mode to all machines in the cluster. If the image is not provided, the latest
 // version of the official Caddy Docker image is used.
-func (cli *Client) NewCaddyDeployment(image, config string, placement api.Placement) (*deploy.Deployment, error) {
+func (c *CaddyClient) NewDeployment(ctx context.Context, opts CaddyDeploymentOptions) (*deploy.Deployment, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	image := opts.Image
 	if image == "" {
-		latest, err := LatestCaddyImage()
+		latest, err := latestCaddyImage(ctx)
 		if err != nil {
 			return nil, fmt.Errorf("look up latest Caddy image: %w", err)
 		}
@@ -35,17 +93,32 @@ func (cli *Client) NewCaddyDeployment(image, config string, placement api.Placem
 
 	spec := api.ServiceSpec{
 		Container: api.ContainerSpec{
-			Command: []string{"caddy", "run", "-c", "/config/Caddyfile"},
 			Env: map[string]string{
 				"CADDY_ADMIN": "unix//run/caddy/admin.sock",
 			},
+			Healthcheck: &api.HealthcheckSpec{
+				Test: []string{
+					"CMD",
+					"curl", "-fsS",
+					"-o", "/dev/null",
+					"--unix-socket", "/run/caddy/admin.sock",
+					"http://localhost/config/",
+				},
+				Interval:      30 * time.Second,
+				Timeout:       5 * time.Second,
+				Retries:       3,
+				StartPeriod:   10 * time.Second,
+				StartInterval: 1 * time.Second,
+			},
 			Image: image,
 			VolumeMounts: []api.VolumeMount{
 				{
 					VolumeName:    "data",
-					ContainerPath: "/config",
+					ContainerPath: "/etc/caddy",
+					ReadOnly:      true,
 				},
 				{
+					// Keep local TLS assets persistent while Caddy uses the default file system storage.
 					VolumeName:    "data",
 					ContainerPath: "/data",
 				},
@@ -62,7 +135,7 @@ func (cli *Client) NewCaddyDeployment(image, config string, placement api.Placem
 		},
 		Mode:      api.ServiceModeGlobal,
 		Name:      CaddyServiceName,
-		Placement: placement,
+		Placement: opts.Placement,
 		Ports: []api.PortSpec{
 			{
 				PublishedPort: 80,
@@ -113,28 +186,44 @@ func (cli *Client) NewCaddyDeployment(image, config string, placement api.Placem
 		},
 	}
 
-	if config != "" {
+	if opts.Config != "" {
 		spec.Caddy = &api.CaddySpec{
-			Config: config,
+			Config: opts.Config,
 		}
 	}
 
-	return cli.NewDeployment(spec, nil), nil
+	return c.client.NewDeployment(spec, nil), nil
 }
 
-// LatestCaddyImage returns the latest image of the official Caddy Docker image on Docker Hub.
+// latestCaddyImage returns the latest image of the official Caddy Docker image on Docker Hub.
 // The latest image is determined by the latest version tag 2.x.x.
-func LatestCaddyImage() (reference.NamedTagged, error) {
+func latestCaddyImage(ctx context.Context) (reference.NamedTagged, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
 	repo, err := name.NewRepository(CaddyImage)
 	if err != nil {
 		return nil, fmt.Errorf("parse image: %w", err)
 	}
-	tags, err := remote.List(repo)
+	tags, err := remote.List(repo, remote.WithContext(ctx))
 	if err != nil {
 		return nil, fmt.Errorf("list image tags: %w", err)
 	}
 
-	// Default to the 'latest' tag but try to find the latest version tag 2.x.x.
+	image, err := reference.ParseDockerRef(CaddyImage)
+	if err != nil {
+		return nil, fmt.Errorf("parse image: %w", err)
+	}
+	imageWithTag, err := reference.WithTag(image, latestCaddyTag(tags))
+	if err != nil {
+		return nil, fmt.Errorf("set image tag: %w", err)
+	}
+
+	return imageWithTag, nil
+}
+
+// latestCaddyTag selects the newest stable 2.x.x tag, falling back to latest.
+func latestCaddyTag(tags []string) string {
 	latestTag := "latest"
 	var latestVersion *semver.Version
 	for _, t := range tags {
@@ -152,14 +241,5 @@ func LatestCaddyImage() (reference.NamedTagged, error) {
 		}
 	}
 
-	image, err := reference.ParseDockerRef(CaddyImage)
+	return latestTag
-	if err != nil {
-		return nil, fmt.Errorf("parse image: %w", err)
-	}
-	imageWithTag, err := reference.WithTag(image, latestTag)
-	if err != nil {
-		return nil, fmt.Errorf("set image tag: %w", err)
-	}
-
-	return imageWithTag, nil
 }

pkg/client/caddy_certificate.go

@@ -0,0 +1,44 @@
+package client
+
+import (
+	"context"
+	"errors"
+
+	"github.com/psviderski/uncloud/pkg/api"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+// CaddyListCertificatesOptions controls which machine's certificate storage replica is queried.
+type CaddyListCertificatesOptions struct {
+	// Machine is the machine name or ID. If empty, the machine the client is connected to is used.
+	Machine string
+}
+
+// ListCertificates lists issued certificates from the Caddy's managed certificate storage backed by the distributed
+// cluster store. It doesn't verify trust or whether Caddy serves them and may include expired certificates.
+// Private key assets are never read or returned.
+//
+// This is a non-atomic inventory of one store replica. It does not wait for replication.
+// It returns parsing errors as a combined error, but still returns successfully read certificates regardless
+// of errors.
+func (c *CaddyClient) ListCertificates(ctx context.Context, opts CaddyListCertificatesOptions) ([]api.IssuedCertificate, error) {
+	if opts.Machine != "" {
+		ctx = ProxySingleMachineContext(ctx, opts.Machine)
+	}
+	resp, err := c.Storage.ListCertificates(ctx, &emptypb.Empty{})
+	if err != nil {
+		return nil, err
+	}
+
+	var certs []api.IssuedCertificate
+	var errs []error
+	for _, p := range resp.Certificates {
+		if cert, err := api.IssuedCertificateFromProto(p); err != nil {
+			errs = append(errs, err)
+		} else {
+			certs = append(certs, cert)
+		}
+	}
+
+	return certs, errors.Join(errs...)
+}

pkg/client/caddy_test.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"testing"
 
 	"github.com/distribution/reference"
@@ -11,8 +12,29 @@ import (
 func TestLatestCaddyImage(t *testing.T) {
 	t.Parallel()
 
-	image, err := LatestCaddyImage()
+	image, err := latestCaddyImage(context.Background())
 	require.NoError(t, err)
 
 	assert.Regexp(t, `^caddy:2\.\d+\.\d+$`, reference.FamiliarString(image))
 }
+
+func TestLatestCaddyTag(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		name string
+		tags []string
+		want string
+	}{
+		{name: "empty", want: "latest"},
+		{name: "semantic ordering", tags: []string{"2.9.9", "2.11.4", "2.10.0", "2.11.3"}, want: "2.11.4"},
+		{name: "ignore other versions and variants", tags: []string{
+			"latest", "1.0.0", "3.0.0", "2.12.0-rc.1", "2.12.0-alpine", "2.12", "v2.12.0", "2.11.4",
+		}, want: "2.11.4"},
+		{name: "no stable version", tags: []string{"builder", "2.12.0-beta.1"}, want: "latest"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, latestCaddyTag(tt.tags))
+		})
+	}
+}

pkg/client/client.go

@@ -25,8 +25,8 @@ type Client struct {
 	//  Methods such as Reset or Inspect are ambiguous in the context of a machine+cluster client.
 	pb.MachineClient
 	pb.ClusterClient
-	Caddy        pb.CaddyClient
+	// Caddy provides Caddy configuration, deployment, and certificate storage operations.
-	CaddyStorage pb.CaddyStorageClient
+	Caddy *CaddyClient
 	// Docker is a namespaced client for the Docker service to distinguish Uncloud-specific service container operations
 	// from generic Docker operations.
 	Docker *docker.Client
@@ -58,8 +58,11 @@ func New(ctx context.Context, connector Connector) (*Client, error) {
 
 	c.MachineClient = pb.NewMachineClient(c.conn)
 	c.ClusterClient = pb.NewClusterClient(c.conn)
-	c.Caddy = pb.NewCaddyClient(c.conn)
+	c.Caddy = &CaddyClient{
-	c.CaddyStorage = pb.NewCaddyStorageClient(c.conn)
+		Storage: pb.NewCaddyStorageClient(c.conn),
+		grpc:    pb.NewCaddyClient(c.conn),
+		client:  c,
+	}
 	c.Docker = docker.NewClient(c.conn)
 	c.leases = distlockgrpc.NewLeaseClient(c.conn)
 

pkg/client/machine.go

@@ -143,7 +143,7 @@ func (cli *Client) WaitClusterReady(ctx context.Context, timeout time.Duration)
 }
 
 // WaitForStoreVersion waits until the cluster store on the target machine has reached each requested actor version
-// in minVersion, with no known missing or pending transactions through those versions.
+// in minVersion, with no known missing or pending transactions through those versions from active members.
 // The context controls cancellation and the deadline. An empty minVersion requires no replication.
 // This method observes replication without initiating synchronisation.
 //
@@ -157,7 +157,7 @@ func (cli *Client) WaitClusterReady(ctx context.Context, timeout time.Duration)
 //
 // Success does not guarantee an exact snapshot or delivery of every historical value.
 // Callers that require a specific record or condition should verify it after waiting.
-func (cli *Client) WaitForStoreVersion(ctx context.Context, minVersion map[string]uint64) error {
+func (cli *Client) WaitForStoreVersion(ctx context.Context, minVersion api.StoreVersion) error {
 	_, err := cli.MachineClient.WaitForStoreVersion(ctx, &pb.WaitForStoreVersionRequest{MinVersion: minVersion})
 	return err
 }

pkg/client/service.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"maps"
 	"slices"
 	"sync"
 
@@ -320,40 +321,18 @@ func (cli *Client) StartService(ctx context.Context, id string) error {
 
 // ListServices returns a list of all services and their containers.
 func (cli *Client) ListServices(ctx context.Context) ([]api.Service, error) {
-	machines, err := cli.ListMachines(ctx, nil)
+	// Broadcast the container list request to all machines in the cluster.
-	if err != nil {
+	listCtx := ProxyMachinesContext(ctx, nil)
-		return nil, fmt.Errorf("list machines: %w", err)
+	// List all containers including stopped ones and deployment hooks.
-	}
-
-	// Broadcast the container list request to all available machines.
-	md := metadata.New(nil)
-	for _, m := range machines {
-		if m.State == pb.MachineMember_UP || m.State == pb.MachineMember_SUSPECT {
-			md.Append("machines", m.Machine.Id)
-		} else {
-			tui.PrintWarning(fmt.Sprintf("failed to list service containers on machine '%s' (state is %s). "+
-				"The results may be incomplete.", m.Machine.Name, m.State.String()))
-		}
-	}
-	listCtx := metadata.NewOutgoingContext(ctx, md)
-
-	// List all containers including stopped ones.
 	opts := container.ListOptions{All: true}
 	machineContainers, err := cli.Docker.ListServiceContainers(listCtx, "", opts)
 	if err != nil {
 		return nil, fmt.Errorf("list containers: %w", err)
 	}
 
-	// TODO: optimise by extracting services from the list of all containers instead of inspecting each service.
+	// Group containers by service ID using the broadcast results.
-	//  Most of the code can be reused in both InspectService and ListServices.
 	servicesByID := make(map[string]api.Service)
 	for _, mc := range machineContainers {
-		// NOTE: Metadata should never be nil in practice. This is legacy fallback that will be removed.
-		if mc.Metadata == nil {
-			tui.PrintWarning("metadata is missing in response from unknown server")
-			continue
-		}
-
 		if mc.Metadata.Error != "" {
 			// TODO: return failed machines in the response.
 			tui.PrintWarning(fmt.Sprintf("failed to list containers on machine '%s': %s",
@@ -362,25 +341,30 @@ func (cli *Client) ListServices(ctx context.Context) ([]api.Service, error) {
 		}
 
 		for _, ctr := range append(mc.Containers, mc.HookContainers...) {
-			if _, ok := servicesByID[ctr.ServiceID()]; ok {
+			serviceID := ctr.ServiceID()
-				continue
+			svc, ok := servicesByID[serviceID]
-			}
+			if !ok {
-
+				svc = api.Service{
-			svc, err := cli.InspectService(ctx, ctr.ServiceID())
+					ID:   serviceID,
-			if err != nil {
+					Name: ctr.ServiceName(),
-				if errors.Is(err, api.ErrNotFound) {
+					Mode: ctr.ServiceMode(),
-					continue
 				}
-				return nil, fmt.Errorf("inspect service: %w", err)
 			}
 
-			servicesByID[ctr.ServiceID()] = svc
+			machineContainer := api.MachineServiceContainer{
+				MachineID:   mc.Metadata.MachineId,
+				MachineName: mc.Metadata.MachineName,
+				Container:   ctr,
+			}
+			if ctr.IsHook() {
+				svc.HookContainers = append(svc.HookContainers, machineContainer)
+			} else {
+				svc.Containers = append(svc.Containers, machineContainer)
+			}
+
+			servicesByID[serviceID] = svc
 		}
 	}
 
-	services := make([]api.Service, 0, len(servicesByID))
+	return slices.Collect(maps.Values(servicesByID)), nil
-	for _, svc := range servicesByID {
-		services = append(services, svc)
-	}
-	return services, nil
 }

test/e2e/cluster_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/psviderski/uncloud/api/pb"
 	"github.com/psviderski/uncloud/internal/ucind"
+	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/psviderski/uncloud/pkg/client"
 	"github.com/psviderski/uncloud/pkg/distlock"
 	"github.com/stretchr/testify/assert"
@@ -195,25 +196,23 @@ func TestClusterLifecycle(t *testing.T) {
 			clients[i] = cli
 		}
 
-		storeVersion := func(clis ...*client.Client) map[string]uint64 {
+		storeVersion := func(clis ...*client.Client) api.StoreVersion {
 			t.Helper()
 			callCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
 			defer cancel()
 
-			version := make(map[string]uint64)
+			version := make(api.StoreVersion)
 			for _, cli := range clis {
-				resp, err := cli.MachineClient.InspectMachine(callCtx, &emptypb.Empty{})
+				resp, err := cli.MachineClient.InspectMachine(callCtx, nil)
 				require.NoError(t, err)
 				require.Len(t, resp.Machines, 1)
 				m := resp.Machines[0]
 				require.Len(t, m.StoreVersion, 3)
-				for actor, v := range m.StoreVersion {
+				version.MergeMax(m.StoreVersion)
-					version[actor] = max(version[actor], v)
-				}
 			}
 			return version
 		}
-		waitForStoreVersion := func(cli *client.Client, version map[string]uint64) {
+		waitForStoreVersion := func(cli *client.Client, version api.StoreVersion) {
 			t.Helper()
 			waitCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
 			defer cancel()
@@ -229,19 +228,19 @@ func TestClusterLifecycle(t *testing.T) {
 		t.Cleanup(func() {
 			cleanupCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
 			defer cancel()
-			_, _ = clients[0].CaddyStorage.Delete(cleanupCtx, &pb.DeleteCaddyStorageRequest{Key: prefix})
+			_, _ = clients[0].Caddy.Storage.Delete(cleanupCtx, &pb.DeleteCaddyStorageRequest{Key: prefix})
-			_, _ = clients[1].CaddyStorage.Delete(cleanupCtx, &pb.DeleteCaddyStorageRequest{Key: prefix})
+			_, _ = clients[1].Caddy.Storage.Delete(cleanupCtx, &pb.DeleteCaddyStorageRequest{Key: prefix})
 		})
 
 		// Create and overwrite a key on the first machine before waiting for replication.
 		var updatedAt time.Time
 		for _, value := range [][]byte{[]byte("test-value"), []byte("replacement-value")} {
-			_, err := clients[0].CaddyStorage.Store(ctx, &pb.StoreCaddyStorageRequest{Key: key, Value: value})
+			_, err := clients[0].Caddy.Storage.Store(ctx, &pb.StoreCaddyStorageRequest{Key: key, Value: value})
 			require.NoError(t, err)
 
 			// A Load through another machine must find the value on the machine that accepted the local write,
 			// regardless of whether Corrosion has replicated it to the other machines yet.
-			loadResp, err := clients[1].CaddyStorage.Load(client.ProxySingleMachineContext(ctx, c.Machines[0].ID),
+			loadResp, err := clients[1].Caddy.Storage.Load(client.ProxySingleMachineContext(ctx, c.Machines[0].ID),
 				&pb.LoadCaddyStorageRequest{Key: key})
 			require.NoError(t, err)
 			require.Equal(t, value, loadResp.Value)
@@ -257,7 +256,7 @@ func TestClusterLifecycle(t *testing.T) {
 
 		// Write a distinct key on the second machine, then capture the combined store version from both machines.
 		otherValue := []byte("second-value")
-		_, err := clients[1].CaddyStorage.Store(ctx, &pb.StoreCaddyStorageRequest{Key: otherKey, Value: otherValue})
+		_, err := clients[1].Caddy.Storage.Store(ctx, &pb.StoreCaddyStorageRequest{Key: otherKey, Value: otherValue})
 		require.NoError(t, err)
 		version := storeVersion(clients[0], clients[1])
 		values := map[string][]byte{key: []byte("replacement-value"), otherKey: otherValue}
@@ -269,7 +268,7 @@ func TestClusterLifecycle(t *testing.T) {
 
 			// Both final values must be readable locally as soon as the wait returns.
 			for k, v := range values {
-				resp, err := cli.CaddyStorage.Load(ctx, &pb.LoadCaddyStorageRequest{Key: k})
+				resp, err := cli.Caddy.Storage.Load(ctx, &pb.LoadCaddyStorageRequest{Key: k})
 				require.NoError(t, err)
 				assert.Equal(t, v, resp.Value)
 				require.NoError(t, resp.UpdatedAt.CheckValid())
@@ -277,7 +276,7 @@ func TestClusterLifecycle(t *testing.T) {
 					assert.True(t, resp.UpdatedAt.AsTime().Equal(updatedAt))
 				}
 
-				statResp, err := cli.CaddyStorage.Stat(ctx, &pb.StatCaddyStorageRequest{Key: k})
+				statResp, err := cli.Caddy.Storage.Stat(ctx, &pb.StatCaddyStorageRequest{Key: k})
 				require.NoError(t, err)
 				assert.Equal(t, k, statResp.Key)
 				assert.True(t, statResp.UpdatedAt.AsTime().Equal(resp.UpdatedAt.AsTime()))
@@ -286,25 +285,25 @@ func TestClusterLifecycle(t *testing.T) {
 			}
 
 			// A path with descendants should exist as a directory even though no value is stored at that key.
-			statResp, err := cli.CaddyStorage.Stat(ctx, &pb.StatCaddyStorageRequest{Key: prefix + "/key"})
+			statResp, err := cli.Caddy.Storage.Stat(ctx, &pb.StatCaddyStorageRequest{Key: prefix + "/key"})
 			require.NoError(t, err)
 			assert.Equal(t, prefix+"/key", statResp.Key)
 			assert.Nil(t, statResp.UpdatedAt)
 			assert.EqualValues(t, 0, statResp.Size)
 			assert.False(t, statResp.IsTerminal)
 
-			listResp, err := cli.CaddyStorage.List(ctx, &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: true})
+			listResp, err := cli.Caddy.Storage.List(ctx, &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: true})
 			require.NoError(t, err)
 			assert.Equal(t, []string{prefix + "/key", key, otherKey}, listResp.Keys)
 
 			// A non-recursive list should only return the immediate child keys.
-			listResp, err = cli.CaddyStorage.List(ctx, &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: false})
+			listResp, err = cli.Caddy.Storage.List(ctx, &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: false})
 			require.NoError(t, err)
 			assert.Equal(t, []string{prefix + "/key", otherKey}, listResp.Keys)
 		}
 
 		// Delete through one machine. The deletion must reach the other replicas through Corrosion.
-		_, err = clients[2].CaddyStorage.Delete(ctx, &pb.DeleteCaddyStorageRequest{Key: prefix})
+		_, err = clients[2].Caddy.Storage.Delete(ctx, &pb.DeleteCaddyStorageRequest{Key: prefix})
 		require.NoError(t, err)
 		version = storeVersion(clients[2])
 
@@ -312,16 +311,16 @@ func TestClusterLifecycle(t *testing.T) {
 			waitForStoreVersion(cli, version)
 
 			for k := range values {
-				_, err = cli.CaddyStorage.Load(ctx, &pb.LoadCaddyStorageRequest{Key: k})
+				_, err = cli.Caddy.Storage.Load(ctx, &pb.LoadCaddyStorageRequest{Key: k})
 				assert.Equal(t, codes.NotFound, status.Code(err))
-				_, err = cli.CaddyStorage.Stat(ctx, &pb.StatCaddyStorageRequest{Key: k})
+				_, err = cli.Caddy.Storage.Stat(ctx, &pb.StatCaddyStorageRequest{Key: k})
 				assert.Equal(t, codes.NotFound, status.Code(err))
 			}
-			_, err = cli.CaddyStorage.List(ctx, &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: true})
+			_, err = cli.Caddy.Storage.List(ctx, &pb.ListCaddyStorageRequest{Prefix: prefix, Recursive: true})
 			assert.Equal(t, codes.NotFound, status.Code(err))
 
 			// Delete is idempotent on each machine.
-			_, err = cli.CaddyStorage.Delete(ctx, &pb.DeleteCaddyStorageRequest{Key: prefix})
+			_, err = cli.Caddy.Storage.Delete(ctx, &pb.DeleteCaddyStorageRequest{Key: prefix})
 			require.NoError(t, err)
 		}
 	})
@@ -352,12 +351,12 @@ func TestClusterLifecycle(t *testing.T) {
 		})
 
 		t.Run("zero version for unknown actor", func(t *testing.T) {
-			err := cli.WaitForStoreVersion(ctx, map[string]uint64{uuid.NewString(): 0})
+			err := cli.WaitForStoreVersion(ctx, api.StoreVersion{uuid.NewString(): 0})
 			require.NoError(t, err)
 		})
 
 		t.Run("invalid actor UUID", func(t *testing.T) {
-			err := cli.WaitForStoreVersion(ctx, map[string]uint64{"not-a-uuid": 1})
+			err := cli.WaitForStoreVersion(ctx, api.StoreVersion{"not-a-uuid": 1})
 			require.Equal(t, codes.InvalidArgument, status.Code(err))
 		})
 
@@ -366,7 +365,7 @@ func TestClusterLifecycle(t *testing.T) {
 			defer cancel()
 
 			// Background writes cannot satisfy a target for an actor that does not exist.
-			err := cli.WaitForStoreVersion(waitCtx, map[string]uint64{uuid.NewString(): 1})
+			err := cli.WaitForStoreVersion(waitCtx, api.StoreVersion{uuid.NewString(): 1})
 			require.Equal(t, codes.DeadlineExceeded, status.Code(err))
 		})
 
@@ -386,7 +385,7 @@ func TestClusterLifecycle(t *testing.T) {
 			timer := time.AfterFunc(500*time.Millisecond, cancel)
 			defer timer.Stop()
 
-			err := cli.WaitForStoreVersion(waitCtx, map[string]uint64{uuid.NewString(): 1})
+			err := cli.WaitForStoreVersion(waitCtx, api.StoreVersion{uuid.NewString(): 1})
 			require.Equal(t, codes.Canceled, status.Code(err))
 		})
 	})

test/e2e/machine_test.go

@@ -459,8 +459,8 @@ func TestMachineOperations(t *testing.T) {
 	t.Run("remove machine clears container records from cluster store", func(t *testing.T) {
 		// The Caddy controller needs a local Caddy container to supply the global config before it can generate
 		// routes. Place it on the connected machine by ID because earlier tests rename that machine.
-		caddyDeployment, err := cli.NewCaddyDeployment("", "", api.Placement{
+		caddyDeployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{
-			Machines: []string{c.Machines[0].ID},
+			Placement: api.Placement{Machines: []string{c.Machines[0].ID}},
 		})
 		require.NoError(t, err)
 		monitorPeriod := 5 * time.Second
@@ -513,7 +513,7 @@ func TestMachineOperations(t *testing.T) {
 
 		// The Caddyfile contains both upstream IPs before the machine removal.
 		require.Eventually(t, func() bool {
-			cfg, err := cli.Caddy.GetConfig(ctx, nil)
+			cfg, err := cli.Caddy.Config(ctx, client.CaddyConfigOptions{})
 			if err != nil {
 				return false
 			}
@@ -528,7 +528,7 @@ func TestMachineOperations(t *testing.T) {
 		// so the Caddy controller regenerates a Caddyfile without that upstream while keeping the
 		// upstreams for the still-running containers.
 		require.Eventually(t, func() bool {
-			cfg, err := cli.Caddy.GetConfig(ctx, nil)
+			cfg, err := cli.Caddy.Config(ctx, client.CaddyConfigOptions{})
 			if err != nil {
 				return false
 			}

test/e2e/service_test.go

@@ -17,7 +17,6 @@ import (
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/go-units"
-	"github.com/psviderski/uncloud/api/pb"
 	"github.com/psviderski/uncloud/internal/machine/metrics"
 	"github.com/psviderski/uncloud/internal/machine/network"
 	"github.com/psviderski/uncloud/internal/secret"
@@ -316,7 +315,7 @@ func TestDeployment(t *testing.T) {
 			}
 		})
 
-		deployment, err := cli.NewCaddyDeployment("", "", api.Placement{})
+		deployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{})
 		require.NoError(t, err)
 		deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
 
@@ -331,7 +330,7 @@ func TestDeployment(t *testing.T) {
 		ctr := svc.Containers[0].Container
 		assert.Regexp(t, `^caddy:2\.\d+\.\d+$`, ctr.Config.Image)
 
-		config, err := cli.Caddy.GetConfig(ctx, nil)
+		config, err := cli.Caddy.Config(ctx, client.CaddyConfigOptions{})
 		require.NoError(t, err)
 
 		assert.Contains(t, config.Caddyfile, "# Caddyfile autogenerated by Uncloud")
@@ -347,8 +346,8 @@ func TestDeployment(t *testing.T) {
 		})
 
 		// Deploy to machine #0.
-		deployment, err := cli.NewCaddyDeployment("", "", api.Placement{
+		deployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{
-			Machines: []string{c.Machines[0].Name},
+			Placement: api.Placement{Machines: []string{c.Machines[0].Name}},
 		})
 		require.NoError(t, err)
 		deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
@@ -366,7 +365,7 @@ func TestDeployment(t *testing.T) {
 		// initialContainerID := svc.Containers[0].Container.ID
 
 		// Deploy to all machines without a placement constraint.
-		deployment, err = cli.NewCaddyDeployment(image, "", api.Placement{})
+		deployment, err = cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Image: image})
 		require.NoError(t, err)
 		deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
 
@@ -401,7 +400,7 @@ func TestDeployment(t *testing.T) {
 
 		// Without a Caddy container, the controller has no global config to pair with application configs. Keep any
 		// previously saved Caddyfile unchanged rather than publishing a new one without Caddy's global settings.
-		savedBefore, savedBeforeErr := cli.Caddy.GetConfig(ctx, nil)
+		savedBefore, savedBeforeErr := cli.Caddy.Config(ctx, client.CaddyConfigOptions{})
 		if savedBeforeErr != nil {
 			require.Equal(t, codes.NotFound, status.Code(savedBeforeErr))
 		}
@@ -432,7 +431,7 @@ func TestDeployment(t *testing.T) {
 		assertServiceMatchesSpec(t, svc, spec)
 
 		require.Never(t, func() bool {
-			current, currentErr := cli.Caddy.GetConfig(ctx, nil)
+			current, currentErr := cli.Caddy.Config(ctx, client.CaddyConfigOptions{})
 			if savedBeforeErr != nil {
 				return currentErr == nil
 			}
@@ -447,7 +446,7 @@ func TestDeployment(t *testing.T) {
 myapp.example.com {
 	reverse_proxy 1.2.3.4:8000
 }`
-		caddyDeployment, err := cli.NewCaddyDeployment("", caddyCaddyfile, api.Placement{})
+		caddyDeployment, err := cli.Caddy.NewDeployment(ctx, client.CaddyDeploymentOptions{Config: caddyCaddyfile})
 		require.NoError(t, err)
 		caddyDeployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
 
@@ -459,9 +458,9 @@ myapp.example.com {
 		assertServiceMatchesSpec(t, caddySvc, caddyDeployment.Spec)
 
 		// Wait for the Caddyfile to be regenerated with both custom configs.
-		var config *pb.GetCaddyConfigResponse
+		var config api.CaddyConfig
 		require.Eventually(t, func() bool {
-			config, err = cli.Caddy.GetConfig(ctx, nil)
+			config, err = cli.Caddy.Config(ctx, client.CaddyConfigOptions{})
 			if err != nil {
 				return false
 			}
@@ -526,7 +525,7 @@ myapp.example.com {
 		time.Sleep(2 * time.Second)
 
 		// Check that the Caddy config hasn't changed.
-		newConfig, err := cli.Caddy.GetConfig(ctx, nil)
+		newConfig, err := cli.Caddy.Config(ctx, client.CaddyConfigOptions{})
 		require.NoError(t, err)
 
 		// Compare stable parts of the Caddyfile only (skip autogenerated comment with timestamp).

website/docs/1-overview.md

@@ -20,7 +20,7 @@ traditional container orchestrators like Kubernetes or Swarm:
 * Scaling services across multiple machines
 * Cross-machine service communication without exposing ports to the internet
 * DNS-based service discovery
-* Automatic HTTPS and reverse proxy configuration
+* Automatic HTTPS and reverse proxy configuration with clustered storage for certificates
 * Load balancing
 * Persistent storage
 

website/docs/3-concepts/2-ingress/1-overview.md

@@ -24,3 +24,11 @@ When you [publish a service port](2-publishing-services.md), Uncloud automatical
 
 For advanced use cases, Uncloud allows to customise the Caddy config using the `x-caddy` extension in Compose files.
 See [Custom Caddy configuration](2-publishing-services.md#custom-caddy-configuration) for details.
+
+:::tip Shared TLS certificates
+
+Uncloud also provides native cluster storage for Caddy. It shares TLS certificates and ACME challenge tokens across
+machines and coordinates certificate issuance. See [Cluster storage for Caddy](4-cluster-storage-for-caddy.md)
+to enable it.
+
+:::

website/docs/3-concepts/2-ingress/3-managing-caddy.md

@@ -3,6 +3,10 @@
 Caddy is automatically deployed as a global service `caddy` when you initialise a cluster with `uc machine init`. By
 default, it runs on every machine to handle incoming HTTP/HTTPS traffic and route it to your services.
 
+Since [v0.21.0](https://github.com/psviderski/uncloud/releases/tag/v0.21.0), Uncloud provides native
+[cluster storage for Caddy](4-cluster-storage-for-caddy.md) (opt-in) to share TLS certificates and coordinate
+certificate issuance across machines.
+
 ## Checking status
 
 View the `caddy` service status and which machines it's running on:
@@ -34,7 +38,7 @@ uc caddy deploy
 Deploy a specific version or custom image:
 
 ```shell
-uc caddy deploy --image caddybuilds/caddy-cloudflare:2.10.2
+uc caddy deploy --image caddybuilds/caddy-cloudflare:2.11.4
 ```
 
 Deploy only to a specific machine or a subset of machines (comma-separated list):
@@ -80,19 +84,27 @@ config that uses the DNS challenge with Cloudflare to obtain a wildcard TLS cert
 ```yaml
 services:
   caddy:
-    image: caddybuilds/caddy-cloudflare:2.10.2
+    image: caddybuilds/caddy-cloudflare:2.11.4
-    command: caddy run -c /config/Caddyfile
     environment:
       # unix// is not a typo. Caddy uses network/address format, not a unix:// URL.
       CADDY_ADMIN: unix//run/caddy/admin.sock
     env_file:
       # Contains CLOUDFLARE_API_TOKEN=xxxxx
       - .env.secrets
+    healthcheck:
+      test: curl -fsS -o /dev/null --unix-socket /run/caddy/admin.sock http://localhost/config/
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+      start_interval: 1s
     volumes:
+      - /var/lib/uncloud/caddy:/etc/caddy:ro
+      # Persists TLS certificates and other assets when using Caddy's default local storage.
       - /var/lib/uncloud/caddy:/data
-      - /var/lib/uncloud/caddy:/config
       - /run/uncloud/caddy:/run/caddy
-      # Required by caddy.storage.uncloud module. Remove this mount if Uncloud cluster storage is not used for Caddy.
+      # Required by the Uncloud cluster storage module (https://github.com/unlabs-dev/caddy-uncloud).
+      # Remove this mount if cluster storage is not used for Caddy.
       # Mount the directory, not the socket file, so Caddy sees a replacement socket after the daemon restarts.
       - /run/uncloud/api:/run/uncloud/api:ro
     x-ports:
@@ -142,9 +154,15 @@ internal.example.com {
 
 :::info note
 
-The specified `command`, `environment`, `volumes`, and `x-ports` properties are essential for Caddy to function
+The specified `environment`, `volumes`, and `x-ports` properties are essential for Caddy to function correctly in the
-correctly in the Uncloud cluster. Do not change the source paths of the volume mounts as the Uncloud daemon relies on
+Uncloud cluster. Do not change the source paths of the volume mounts as the Uncloud daemon relies on them to communicate
-them to communicate with Caddy and update its configuration.
+with Caddy and update its configuration.
+
+The image must include `curl` for the healthcheck and start Caddy with `/etc/caddy/Caddyfile`. Images based on the
+official [Caddy image](https://hub.docker.com/_/caddy) do both by default.
+
+Keep the `/data` mount while using Caddy's default local storage so TLS certificates survive container updates. You can
+remove this mount when using [cluster storage for Caddy](4-cluster-storage-for-caddy.md).
 
 :::
 

website/docs/3-concepts/2-ingress/4-cluster-storage-for-caddy.md

@@ -0,0 +1,58 @@
+# Cluster storage for Caddy
+
+The [Uncloud storage module](https://github.com/unlabs-dev/caddy-uncloud) lets Caddy instances share TLS certificates,
+private keys, and ACME challenge tokens through Uncloud's cluster store. It uses distributed locks to coordinate
+certificate issuance. Once one instance obtains a certificate, the others can use it too.
+
+## Why use cluster storage?
+
+By default, each Caddy instance stores its certificates locally. When DNS points to multiple machines or a load balancer
+distributes traffic between them, an ACME challenge can reach a different instance from the one requesting the
+certificate. That instance may not have the challenge token, which can delay or prevent certificate issuance.
+
+Cluster storage is optional. The default Caddy image does not include the module, so you need to deploy an image that
+includes it and configure Caddy to use it.
+
+## Enabling cluster storage
+
+:::info Requirements
+
+Cluster storage requires Uncloud **v0.21.0 or newer** for both the `uc` CLI and the daemon on every cluster machine.
+Check your CLI version with `uc version` and daemon versions with `uc machine ls`. Upgrade older versions before
+enabling cluster storage.
+
+:::
+
+Create a global Caddyfile, or add `storage uncloud` to the global options in your existing one:
+
+```caddyfile title="global.Caddyfile"
+{
+    storage uncloud
+
+    # Uncomment to enable debug logs useful for troubleshooting storage operations:
+    # debug
+}
+```
+
+Deploy Caddy with the pre-built module image and your global config:
+
+```shell
+uc caddy deploy --image ghcr.io/unlabs-dev/caddy-uncloud:0.1.1 --caddyfile global.Caddyfile
+```
+
+See the [module README](https://github.com/unlabs-dev/caddy-uncloud#usage) for custom image builds, Compose deployment,
+and additional storage options.
+
+## Verifying storage
+
+Check that `storage uncloud` appears in the Caddy config:
+
+```shell
+uc caddy config
+```
+
+List issued certificates in cluster storage with [`uc caddy cert ls`](../../9-cli-reference/uc_caddy_cert_ls.md):
+
+```shell
+uc caddy cert ls
+```

website/docs/9-cli-reference/uc_caddy.md

@@ -20,6 +20,7 @@ Manage Caddy reverse proxy service.
 ## See also
 
 * [uc](uc.md)	 - A CLI tool for managing Uncloud resources such as machines, services, and volumes.
+* [uc caddy cert](uc_caddy_cert.md)	 - Inspect certificates in cluster storage for Caddy.
 * [uc caddy config](uc_caddy_config.md)	 - Show the current Caddy configuration (Caddyfile).
 * [uc caddy deploy](uc_caddy_deploy.md)	 - Deploy or upgrade Caddy reverse proxy across all machines in the cluster.
 * [uc caddy logs](uc_caddy_logs.md)	 - View caddy logs.

website/docs/9-cli-reference/uc_caddy_cert.md

@@ -0,0 +1,24 @@
+# uc caddy cert
+
+Inspect certificates in cluster storage for Caddy.
+
+## Options
+
+```
+  -h, --help   help for cert
+```
+
+## Options inherited from parent commands
+
+```
+      --connect string          Connect to a remote cluster machine without using the Uncloud configuration file. [$UNCLOUD_CONNECT]
+                                Format: [ssh://]user@host[:port], ssh+go://user@host[:port], tcp://host:port, or unix:///path/to/uncloud.sock
+  -c, --context string          Name of the cluster context to use (default is the current context). [$UNCLOUD_CONTEXT]
+      --uncloud-config string   Path to the Uncloud configuration file. [$UNCLOUD_CONFIG] (default "~/.config/uncloud/config.yaml")
+```
+
+## See also
+
+* [uc caddy](uc_caddy.md)	 - Manage Caddy reverse proxy service.
+* [uc caddy cert ls](uc_caddy_cert_ls.md)	 - List certificates in cluster storage for Caddy.
+

website/docs/9-cli-reference/uc_caddy_cert_ls.md

@@ -0,0 +1,38 @@
+# uc caddy cert ls
+
+List certificates in cluster storage for Caddy.
+
+## Synopsis
+
+List certificates stored in the Uncloud cluster storage for Caddy.
+
+Caddy must use the Uncloud storage module (https://github.com/unlabs-dev/caddy-uncloud)
+configured with 'storage uncloud' in the global options for its certificates to appear here.
+
+This inventory does not check whether Caddy currently serves or trusts a certificate.
+
+```
+uc caddy cert ls [flags]
+```
+
+## Options
+
+```
+  -h, --help             help for ls
+  -m, --machine string   Name or ID of the machine to read certificate storage from. (default is connected machine)
+  -o, --output string    Output format: 'json' or empty for a human-readable table.
+```
+
+## Options inherited from parent commands
+
+```
+      --connect string          Connect to a remote cluster machine without using the Uncloud configuration file. [$UNCLOUD_CONNECT]
+                                Format: [ssh://]user@host[:port], ssh+go://user@host[:port], tcp://host:port, or unix:///path/to/uncloud.sock
+  -c, --context string          Name of the cluster context to use (default is the current context). [$UNCLOUD_CONTEXT]
+      --uncloud-config string   Path to the Uncloud configuration file. [$UNCLOUD_CONFIG] (default "~/.config/uncloud/config.yaml")
+```
+
+## See also
+
+* [uc caddy cert](uc_caddy_cert.md)	 - Inspect certificates in cluster storage for Caddy.
+