fix(caddy): config generation for multiple https containers

internal/machine/caddyfile/config.go

@@ -0,0 +1,163 @@
+package caddyfile
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
+	"log/slog"
+	"maps"
+	"net"
+	"net/http"
+	"slices"
+	"strconv"
+	"uncloud/internal/api"
+	"uncloud/internal/machine/docker"
+)
+
+func GenerateConfig(containers []api.Container, verifyResponse string) (*caddy.Config, error) {
+	// Maps hostnames to lists of upstreams (container IP:port pairs).
+	httpHostUpstreams := make(map[string][]string)
+	httpsHostUpstreams := make(map[string][]string)
+	for _, ctr := range containers {
+		logger := slog.With("container", ctr.ID)
+		network, ok := ctr.NetworkSettings.Networks[docker.NetworkName]
+		if !ok {
+			// Container is not connected to the uncloud Docker network (could be host network).
+			continue
+		}
+		if network.IPAddress == "" {
+			logger.Error("Container has no IPv4 address.")
+			continue
+		}
+
+		ports, err := ctr.ServicePorts()
+		if err != nil {
+			logger.Error("Failed to parse service ports for container.", "err", err)
+			continue
+		}
+
+		for _, port := range ports {
+			switch port.Protocol {
+			case api.ProtocolHTTP:
+				upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort)))
+				httpHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream)
+			case api.ProtocolHTTPS:
+				upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort)))
+				httpsHostUpstreams[port.Hostname] = append(httpsHostUpstreams[port.Hostname], upstream)
+			default:
+				// TODO: implement L4 ingress routing for TCP and UDP.
+				logger.Error("Unsupported protocol for ingress port.", "port", port)
+				continue
+			}
+		}
+	}
+
+	var warnings []caddyconfig.Warning
+	servers := make(map[string]*caddyhttp.Server)
+	servers["http"] = &caddyhttp.Server{
+		Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)},
+		Routes: append(
+			hostUpstreamsToRoutes(httpHostUpstreams, &warnings),
+			// Add a route to respond with a static verification response at the /.uncloud-verify path.
+			verificationRoute(verifyResponse, &warnings),
+		),
+	}
+	servers["https"] = &caddyhttp.Server{
+		Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)},
+		Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings),
+	}
+
+	httpApp := caddyhttp.App{
+		Servers: servers,
+	}
+	config := &caddy.Config{
+		AppsRaw: caddy.ModuleMap{
+			"http": caddyconfig.JSON(httpApp, &warnings),
+		},
+	}
+
+	var err error
+	if len(warnings) > 0 {
+		// warnings only contains errors from JSON marshaling, which are highly unlikely with correct code.
+		for _, w := range warnings {
+			err = errors.Join(err, errors.New(w.Message))
+		}
+		return nil, fmt.Errorf("marshal Caddy configuration: %w", err)
+	}
+
+	return config, nil
+}
+
+// hostUpstreamsToRoutes converts a map of hostnames to upstreams to a list of Caddy routes.
+func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyconfig.Warning) []caddyhttp.Route {
+	// Sort hostnames for deterministic output.
+	hostnames := slices.Collect(maps.Keys(hostUpstreams))
+	slices.Sort(hostnames)
+
+	routes := make([]caddyhttp.Route, 0, len(hostUpstreams))
+	for _, hostname := range hostnames {
+		upstreams := hostUpstreams[hostname]
+		upstreamPool := make([]*reverseproxy.Upstream, len(upstreams))
+		for i, upstream := range upstreams {
+			upstreamPool[i] = &reverseproxy.Upstream{
+				Dial: upstream,
+			}
+		}
+		handler := &reverseproxy.Handler{
+			Upstreams: upstreamPool,
+		}
+
+		routes = append(routes, caddyhttp.Route{
+			MatcherSetsRaw: caddyhttp.RawMatcherSets{
+				{
+					"host": caddyconfig.JSON(caddyhttp.MatchHost{hostname}, warnings),
+				},
+			},
+			HandlersRaw: []json.RawMessage{
+				caddyconfig.JSONModuleObject(handler, "handler", "reverse_proxy", warnings),
+			},
+		})
+	}
+	return routes
+}
+
+// verificationRoute returns a Caddy route that responds with the given static response at the /.uncloud-verify path.
+func verificationRoute(response string, warnings *[]caddyconfig.Warning) caddyhttp.Route {
+	// Return the following route:
+	// {
+	//   "match": [
+	//     {
+	//       "path": [
+	//         "/.uncloud-verify"
+	//       ]
+	//     }
+	//   ],
+	//   "handle": [
+	//     {
+	//       "handler": "static_response",
+	//       "body": "<response>",
+	//       "status_code": 200
+	//     }
+	//   ]
+	// }
+
+	staticResponse := caddyhttp.StaticResponse{
+		StatusCode: caddyhttp.WeakString(strconv.Itoa(http.StatusOK)),
+		Body:       response,
+	}
+
+	return caddyhttp.Route{
+		MatcherSetsRaw: caddyhttp.RawMatcherSets{
+			{
+				"path": caddyconfig.JSON(caddyhttp.MatchPath{VerifyPath}, warnings),
+			},
+		},
+		HandlersRaw: []json.RawMessage{
+			caddyconfig.JSONModuleObject(staticResponse, "handler", "static_response", warnings),
+		},
+	}
+}

internal/machine/caddyfile/config_test.go

@@ -0,0 +1,365 @@
+package caddyfile
+
+import (
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"strings"
+	"testing"
+	"uncloud/internal/api"
+	"uncloud/internal/machine/docker"
+)
+
+func TestGenerateConfig(t *testing.T) {
+	tests := []struct {
+		name       string
+		containers []api.Container
+		want       string
+		wantErr    bool
+	}{
+		{
+			name:       "empty containers",
+			containers: []api.Container{},
+			want: `{
+                "servers": {
+                    "http": {
+                        "listen": [":80"],
+                        "routes": [{
+                            "match": [{"path": ["/.uncloud-verify"]}],
+                            "handle": [{
+								"body": "verification-response-body",
+								"handler": "static_response",
+								"status_code": 200
+							}]
+                        }]
+                    },
+                    "https": {
+                        "listen": [":443"]
+                    }
+				}
+			}`,
+			wantErr: false,
+		},
+
+		{
+			name: "HTTP container",
+			containers: []api.Container{
+				newContainer("10.210.0.2", "app.example.com:8080/http"),
+			},
+			want: `{
+                "servers": {
+                    "http": {
+                        "listen": [":80"],
+                        "routes": [
+                            {
+                                "match": [{"host": ["app.example.com"]}],
+                                "handle": [{
+                                    "handler": "reverse_proxy",
+                                    "upstreams": [{"dial": "10.210.0.2:8080"}]
+                                }]
+                            },
+                            {
+                                "match": [{"path": ["/.uncloud-verify"]}],
+                                "handle": [{
+                                    "body": "verification-response-body",
+                                    "handler": "static_response",
+                                    "status_code": 200
+                                }]
+                            }
+                        ]
+                    },
+                    "https": {
+                        "listen": [":443"]
+                    }
+                }
+            }`,
+			wantErr: false,
+		},
+		{
+			name: "load balancing multiple containers",
+			containers: []api.Container{
+				newContainer("10.210.0.2", "app.example.com:8080/http"),
+				newContainer("10.210.0.3", "app.example.com:8080/http"),
+			},
+			want: `{
+                "servers": {
+                    "http": {
+                        "listen": [":80"],
+                        "routes": [
+                            {
+                                "match": [{"host": ["app.example.com"]}],
+                                "handle": [{
+                                    "handler": "reverse_proxy",
+                                    "upstreams": [
+                                        {"dial": "10.210.0.2:8080"},
+                                        {"dial": "10.210.0.3:8080"}
+                                    ]
+                                }]
+                            },
+                            {
+                                "match": [{"path": ["/.uncloud-verify"]}],
+                                "handle": [{
+                                    "body": "verification-response-body",
+                                    "handler": "static_response",
+                                    "status_code": 200
+                                }]
+                            }
+                        ]
+                    },
+                    "https": {
+                        "listen": [":443"]
+                    }
+                }
+            }`,
+			wantErr: false,
+		},
+		{
+			name: "HTTPS container",
+			containers: []api.Container{
+				newContainer("10.210.0.2", "secure.example.com:8000/https"),
+			},
+			want: `{
+                "servers": {
+                    "http": {
+                        "listen": [":80"],
+                        "routes": [
+                            {
+                                "match": [{"path": ["/.uncloud-verify"]}],
+                                "handle": [{
+                                    "body": "verification-response-body",
+                                    "handler": "static_response",
+                                    "status_code": 200
+                                }]
+                            }
+                        ]
+                    },
+                    "https": {
+                        "listen": [":443"],
+                        "routes": [
+                            {
+                                "match": [{"host": ["secure.example.com"]}],
+                                "handle": [{
+                                    "handler": "reverse_proxy",
+                                    "upstreams": [{"dial": "10.210.0.2:8000"}]
+                                }]
+                            }
+                        ]
+                    }
+                }
+            }`,
+			wantErr: false,
+		},
+		{
+			name: "mixed HTTP and HTTPS",
+			containers: []api.Container{
+				newContainer("10.210.0.2",
+					"app.example.com:8080/http",
+					"web.example.com:8000/http"),
+				newContainer("10.210.0.3",
+					"app.example.com:8080/http",
+					"secure.example.com:8888/https"),
+				newContainer("10.210.0.4",
+					"web.example.com:8000/http",
+					"secure.example.com:8888/https"),
+				newContainer("10.210.0.5",
+					"app.example.com:8080/http",
+					"web.example.com:8000/http",
+					"secure.example.com:8888/https"),
+			},
+			want: `{
+                "servers": {
+                    "http": {
+                        "listen": [":80"],
+                        "routes": [
+                            {
+                                "match": [{"host": ["app.example.com"]}],
+                                "handle": [{
+                                    "handler": "reverse_proxy",
+                                    "upstreams": [
+										{"dial": "10.210.0.2:8080"},
+										{"dial": "10.210.0.3:8080"},
+										{"dial": "10.210.0.5:8080"}
+									]
+                                }]
+                            },
+							{
+                                "match": [{"host": ["web.example.com"]}],
+                                "handle": [{
+                                    "handler": "reverse_proxy",
+                                    "upstreams": [
+										{"dial": "10.210.0.2:8000"},
+										{"dial": "10.210.0.4:8000"},
+										{"dial": "10.210.0.5:8000"}
+									]
+                                }]
+                            },
+                            {
+                                "match": [{"path": ["/.uncloud-verify"]}],
+                                "handle": [{
+                                    "body": "verification-response-body",
+                                    "handler": "static_response",
+                                    "status_code": 200
+                                }]
+                            }
+                        ]
+                    },
+                    "https": {
+                        "listen": [":443"],
+                        "routes": [
+                            {
+                                "match": [{"host": ["secure.example.com"]}],
+                                "handle": [{
+                                    "handler": "reverse_proxy",
+                                    "upstreams": [
+										{"dial": "10.210.0.3:8888"},
+										{"dial": "10.210.0.4:8888"},
+										{"dial": "10.210.0.5:8888"}
+									]
+                                }]
+                            }
+                        ]
+                    }
+                }
+            }`,
+			wantErr: false,
+		},
+		{
+			name: "container without uncloud network ignored",
+			containers: []api.Container{
+				newContainerWithoutNetwork("ignored.example.com:8080/http"),
+			},
+			want: `{
+                "servers": {
+                    "http": {
+                        "listen": [":80"],
+                        "routes": [{
+                            "match": [{"path": ["/.uncloud-verify"]}],
+                            "handle": [{
+                                "body": "verification-response-body",
+                                "handler": "static_response",
+                                "status_code": 200
+                            }]
+                        }]
+                    },
+                    "https": {
+                        "listen": [":443"]
+                    }
+                }
+            }`,
+			wantErr: false,
+		},
+		{
+			name: "container with invalid port ignored",
+			containers: []api.Container{
+				newContainer("10.210.0.2", "invalid-port"),
+			},
+			want: `{
+                "servers": {
+                    "http": {
+                        "listen": [":80"],
+                        "routes": [{
+                            "match": [{"path": ["/.uncloud-verify"]}],
+                            "handle": [{
+                                "body": "verification-response-body",
+                                "handler": "static_response",
+                                "status_code": 200
+                            }]
+                        }]
+                    },
+                    "https": {
+                        "listen": [":443"]
+                    }
+                }
+            }`,
+			wantErr: false,
+		},
+		{
+			name: "containers with unsupported protocols ignored",
+			containers: []api.Container{
+				newContainer("10.210.0.2", "5000/tcp"),
+				newContainer("10.210.0.3", "5000/udp"),
+				newContainer("10.210.0.4", "80:8080/tcp@host"),
+			},
+			want: `{
+                "servers": {
+                    "http": {
+                        "listen": [":80"],
+                        "routes": [{
+                            "match": [{"path": ["/.uncloud-verify"]}],
+                            "handle": [{
+                                "body": "verification-response-body",
+                                "handler": "static_response",
+                                "status_code": 200
+                            }]
+                        }]
+                    },
+                    "https": {
+                        "listen": [":443"]
+                    }
+                }
+            }`,
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config, err := GenerateConfig(tt.containers, "verification-response-body")
+
+			if tt.wantErr {
+				assert.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			require.Len(t, config.AppsRaw, 1, "Expected one http app")
+			require.Contains(t, config.AppsRaw, "http", "Expected http app")
+
+			assert.JSONEq(t, tt.want, string(config.AppsRaw["http"]), "Generated Caddy app config doesn't match")
+		})
+	}
+}
+
+func newContainer(ip string, ports ...string) api.Container {
+	portsLabel := strings.Join(ports, ",")
+	return api.Container{
+		ContainerJSON: types.ContainerJSON{
+			ContainerJSONBase: &types.ContainerJSONBase{},
+			NetworkSettings: &types.NetworkSettings{
+				Networks: map[string]*network.EndpointSettings{
+					docker.NetworkName: {
+						IPAddress: ip,
+					},
+				},
+			},
+			Config: &container.Config{
+				Labels: map[string]string{
+					api.LabelServicePorts: portsLabel,
+				},
+			},
+		},
+	}
+}
+
+func newContainerWithoutNetwork(ports ...string) api.Container {
+	portsLabel := strings.Join(ports, ",")
+	return api.Container{
+		ContainerJSON: types.ContainerJSON{
+			ContainerJSONBase: &types.ContainerJSONBase{},
+			NetworkSettings: &types.NetworkSettings{
+				Networks: map[string]*network.EndpointSettings{
+					"other-network": {
+						IPAddress: "172.17.0.2",
+					},
+				},
+			},
+			Config: &container.Config{
+				Labels: map[string]string{
+					api.LabelServicePorts: portsLabel,
+				},
+			},
+		},
+	}
+}

internal/machine/caddyfile/controller.go

@@ -3,21 +3,12 @@ package caddyfile
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"github.com/caddyserver/caddy/v2"
-	"github.com/caddyserver/caddy/v2/caddyconfig"
-	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
-	"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
 	"log/slog"
-	"net"
-	"net/http"
 	"os"
 	"path/filepath"
-	"strconv"
 	"uncloud/internal/api"
 	"uncloud/internal/fs"
-	"uncloud/internal/machine/docker"
 	"uncloud/internal/machine/store"
 )
 
@@ -80,6 +71,9 @@ func (c *Controller) Run(ctx context.Context) error {
 				continue
 			}
 			containers, err = c.filterAvailableContainers(containerRecords)
+			for _, c := range containers {
+				fmt.Printf("## caddyfile container: %+v\n", c)
+			}
 			if err != nil {
 				slog.Error("Failed to filter available containers.", "err", err)
 				continue
@@ -107,76 +101,9 @@ func (c *Controller) filterAvailableContainers(containerRecords []store.Containe
 }
 
 func (c *Controller) generateConfig(containers []api.Container) error {
-	// Maps hostnames to lists of upstreams (container IP:port pairs).
+	config, err := GenerateConfig(containers, c.verifyResponse)
-	httpHostUpstreams := make(map[string][]string)
+	if err != nil {
-	httpsHostUpstreams := make(map[string][]string)
+		return err
-	for _, ctr := range containers {
-		logger := slog.With("container", ctr.ID)
-		network, ok := ctr.NetworkSettings.Networks[docker.NetworkName]
-		if !ok {
-			// Container is not connected to the uncloud Docker network (could be host network).
-			continue
-		}
-		if network.IPAddress == "" {
-			logger.Error("Container has no IPv4 address.")
-			continue
-		}
-
-		ports, err := ctr.ServicePorts()
-		if err != nil {
-			logger.Error("Failed to parse service ports for container.", "err", err)
-			continue
-		}
-
-		for _, port := range ports {
-			switch port.Protocol {
-			case api.ProtocolHTTP:
-				upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort)))
-				httpHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream)
-			case api.ProtocolHTTPS:
-				upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort)))
-				httpsHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream)
-			default:
-				if port.Mode == api.PortModeIngress {
-					// TODO: implement L4 ingress routing for TCP and UDP.
-					logger.Error("Unsupported protocol for ingress port.", "port", port)
-					continue
-				}
-			}
-		}
-	}
-
-	var warnings []caddyconfig.Warning
-	servers := make(map[string]*caddyhttp.Server)
-	servers["http"] = &caddyhttp.Server{
-		Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)},
-		Routes: append(
-			hostUpstreamsToRoutes(httpHostUpstreams, &warnings),
-			// Add a route to respond with a static verification response at the /.uncloud-verify path.
-			verificationRoute(c.verifyResponse, &warnings),
-		),
-	}
-	servers["https"] = &caddyhttp.Server{
-		Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)},
-		Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings),
-	}
-
-	httpApp := caddyhttp.App{
-		Servers: servers,
-	}
-	config := &caddy.Config{
-		AppsRaw: caddy.ModuleMap{
-			"http": caddyconfig.JSON(httpApp, &warnings),
-		},
-	}
-
-	var err error
-	if len(warnings) > 0 {
-		// warnings only contains errors from JSON marshaling, which are highly unlikely with correct code.
-		for _, w := range warnings {
-			err = errors.Join(err, errors.New(w.Message))
-		}
-		return fmt.Errorf("marshal Caddy configuration: %w", err)
 	}
 
 	configBytes, err := json.MarshalIndent(config, "", "    ")
@@ -193,68 +120,3 @@ func (c *Controller) generateConfig(containers []api.Container) error {
 
 	return nil
 }
-
-// hostUpstreamsToRoutes converts a map of hostnames to upstreams to a list of Caddy routes.
-func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyconfig.Warning) []caddyhttp.Route {
-	routes := make([]caddyhttp.Route, 0, len(hostUpstreams))
-	for hostname, upstreams := range hostUpstreams {
-		upstreamPool := make([]*reverseproxy.Upstream, len(upstreams))
-		for i, upstream := range upstreams {
-			upstreamPool[i] = &reverseproxy.Upstream{
-				Dial: upstream,
-			}
-		}
-		handler := &reverseproxy.Handler{
-			Upstreams: upstreamPool,
-		}
-
-		routes = append(routes, caddyhttp.Route{
-			MatcherSetsRaw: caddyhttp.RawMatcherSets{
-				{
-					"host": caddyconfig.JSON(caddyhttp.MatchHost{hostname}, warnings),
-				},
-			},
-			HandlersRaw: []json.RawMessage{
-				caddyconfig.JSONModuleObject(handler, "handler", "reverse_proxy", warnings),
-			},
-		})
-	}
-	return routes
-}
-
-// verificationRoute returns a Caddy route that responds with the given static response at the /.uncloud-verify path.
-func verificationRoute(response string, warnings *[]caddyconfig.Warning) caddyhttp.Route {
-	// Return the following route:
-	// {
-	//   "match": [
-	//     {
-	//       "path": [
-	//         "/.uncloud-verify"
-	//       ]
-	//     }
-	//   ],
-	//   "handle": [
-	//     {
-	//       "handler": "static_response",
-	//       "body": "<response>",
-	//       "status_code": 200
-	//     }
-	//   ]
-	// }
-
-	staticResponse := caddyhttp.StaticResponse{
-		StatusCode: caddyhttp.WeakString(strconv.Itoa(http.StatusOK)),
-		Body:       response,
-	}
-
-	return caddyhttp.Route{
-		MatcherSetsRaw: caddyhttp.RawMatcherSets{
-			{
-				"path": caddyconfig.JSON(caddyhttp.MatchPath{VerifyPath}, warnings),
-			},
-		},
-		HandlersRaw: []json.RawMessage{
-			caddyconfig.JSONModuleObject(staticResponse, "handler", "static_response", warnings),
-		},
-	}
-}