diff --git a/internal/machine/caddyfile/config.go b/internal/machine/caddyfile/config.go new file mode 100644 index 00000000..ec7143a9 --- /dev/null +++ b/internal/machine/caddyfile/config.go @@ -0,0 +1,163 @@ +package caddyfile + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy" + "log/slog" + "maps" + "net" + "net/http" + "slices" + "strconv" + "uncloud/internal/api" + "uncloud/internal/machine/docker" +) + +func GenerateConfig(containers []api.Container, verifyResponse string) (*caddy.Config, error) { + // Maps hostnames to lists of upstreams (container IP:port pairs). + httpHostUpstreams := make(map[string][]string) + httpsHostUpstreams := make(map[string][]string) + for _, ctr := range containers { + logger := slog.With("container", ctr.ID) + network, ok := ctr.NetworkSettings.Networks[docker.NetworkName] + if !ok { + // Container is not connected to the uncloud Docker network (could be host network). + continue + } + if network.IPAddress == "" { + logger.Error("Container has no IPv4 address.") + continue + } + + ports, err := ctr.ServicePorts() + if err != nil { + logger.Error("Failed to parse service ports for container.", "err", err) + continue + } + + for _, port := range ports { + switch port.Protocol { + case api.ProtocolHTTP: + upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort))) + httpHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream) + case api.ProtocolHTTPS: + upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort))) + httpsHostUpstreams[port.Hostname] = append(httpsHostUpstreams[port.Hostname], upstream) + default: + // TODO: implement L4 ingress routing for TCP and UDP. + logger.Error("Unsupported protocol for ingress port.", "port", port) + continue + } + } + } + + var warnings []caddyconfig.Warning + servers := make(map[string]*caddyhttp.Server) + servers["http"] = &caddyhttp.Server{ + Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)}, + Routes: append( + hostUpstreamsToRoutes(httpHostUpstreams, &warnings), + // Add a route to respond with a static verification response at the /.uncloud-verify path. + verificationRoute(verifyResponse, &warnings), + ), + } + servers["https"] = &caddyhttp.Server{ + Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)}, + Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings), + } + + httpApp := caddyhttp.App{ + Servers: servers, + } + config := &caddy.Config{ + AppsRaw: caddy.ModuleMap{ + "http": caddyconfig.JSON(httpApp, &warnings), + }, + } + + var err error + if len(warnings) > 0 { + // warnings only contains errors from JSON marshaling, which are highly unlikely with correct code. + for _, w := range warnings { + err = errors.Join(err, errors.New(w.Message)) + } + return nil, fmt.Errorf("marshal Caddy configuration: %w", err) + } + + return config, nil +} + +// hostUpstreamsToRoutes converts a map of hostnames to upstreams to a list of Caddy routes. +func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyconfig.Warning) []caddyhttp.Route { + // Sort hostnames for deterministic output. + hostnames := slices.Collect(maps.Keys(hostUpstreams)) + slices.Sort(hostnames) + + routes := make([]caddyhttp.Route, 0, len(hostUpstreams)) + for _, hostname := range hostnames { + upstreams := hostUpstreams[hostname] + upstreamPool := make([]*reverseproxy.Upstream, len(upstreams)) + for i, upstream := range upstreams { + upstreamPool[i] = &reverseproxy.Upstream{ + Dial: upstream, + } + } + handler := &reverseproxy.Handler{ + Upstreams: upstreamPool, + } + + routes = append(routes, caddyhttp.Route{ + MatcherSetsRaw: caddyhttp.RawMatcherSets{ + { + "host": caddyconfig.JSON(caddyhttp.MatchHost{hostname}, warnings), + }, + }, + HandlersRaw: []json.RawMessage{ + caddyconfig.JSONModuleObject(handler, "handler", "reverse_proxy", warnings), + }, + }) + } + return routes +} + +// verificationRoute returns a Caddy route that responds with the given static response at the /.uncloud-verify path. +func verificationRoute(response string, warnings *[]caddyconfig.Warning) caddyhttp.Route { + // Return the following route: + // { + // "match": [ + // { + // "path": [ + // "/.uncloud-verify" + // ] + // } + // ], + // "handle": [ + // { + // "handler": "static_response", + // "body": "", + // "status_code": 200 + // } + // ] + // } + + staticResponse := caddyhttp.StaticResponse{ + StatusCode: caddyhttp.WeakString(strconv.Itoa(http.StatusOK)), + Body: response, + } + + return caddyhttp.Route{ + MatcherSetsRaw: caddyhttp.RawMatcherSets{ + { + "path": caddyconfig.JSON(caddyhttp.MatchPath{VerifyPath}, warnings), + }, + }, + HandlersRaw: []json.RawMessage{ + caddyconfig.JSONModuleObject(staticResponse, "handler", "static_response", warnings), + }, + } +} diff --git a/internal/machine/caddyfile/config_test.go b/internal/machine/caddyfile/config_test.go new file mode 100644 index 00000000..1d352d0c --- /dev/null +++ b/internal/machine/caddyfile/config_test.go @@ -0,0 +1,365 @@ +package caddyfile + +import ( + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "strings" + "testing" + "uncloud/internal/api" + "uncloud/internal/machine/docker" +) + +func TestGenerateConfig(t *testing.T) { + tests := []struct { + name string + containers []api.Container + want string + wantErr bool + }{ + { + name: "empty containers", + containers: []api.Container{}, + want: `{ + "servers": { + "http": { + "listen": [":80"], + "routes": [{ + "match": [{"path": ["/.uncloud-verify"]}], + "handle": [{ + "body": "verification-response-body", + "handler": "static_response", + "status_code": 200 + }] + }] + }, + "https": { + "listen": [":443"] + } + } + }`, + wantErr: false, + }, + + { + name: "HTTP container", + containers: []api.Container{ + newContainer("10.210.0.2", "app.example.com:8080/http"), + }, + want: `{ + "servers": { + "http": { + "listen": [":80"], + "routes": [ + { + "match": [{"host": ["app.example.com"]}], + "handle": [{ + "handler": "reverse_proxy", + "upstreams": [{"dial": "10.210.0.2:8080"}] + }] + }, + { + "match": [{"path": ["/.uncloud-verify"]}], + "handle": [{ + "body": "verification-response-body", + "handler": "static_response", + "status_code": 200 + }] + } + ] + }, + "https": { + "listen": [":443"] + } + } + }`, + wantErr: false, + }, + { + name: "load balancing multiple containers", + containers: []api.Container{ + newContainer("10.210.0.2", "app.example.com:8080/http"), + newContainer("10.210.0.3", "app.example.com:8080/http"), + }, + want: `{ + "servers": { + "http": { + "listen": [":80"], + "routes": [ + { + "match": [{"host": ["app.example.com"]}], + "handle": [{ + "handler": "reverse_proxy", + "upstreams": [ + {"dial": "10.210.0.2:8080"}, + {"dial": "10.210.0.3:8080"} + ] + }] + }, + { + "match": [{"path": ["/.uncloud-verify"]}], + "handle": [{ + "body": "verification-response-body", + "handler": "static_response", + "status_code": 200 + }] + } + ] + }, + "https": { + "listen": [":443"] + } + } + }`, + wantErr: false, + }, + { + name: "HTTPS container", + containers: []api.Container{ + newContainer("10.210.0.2", "secure.example.com:8000/https"), + }, + want: `{ + "servers": { + "http": { + "listen": [":80"], + "routes": [ + { + "match": [{"path": ["/.uncloud-verify"]}], + "handle": [{ + "body": "verification-response-body", + "handler": "static_response", + "status_code": 200 + }] + } + ] + }, + "https": { + "listen": [":443"], + "routes": [ + { + "match": [{"host": ["secure.example.com"]}], + "handle": [{ + "handler": "reverse_proxy", + "upstreams": [{"dial": "10.210.0.2:8000"}] + }] + } + ] + } + } + }`, + wantErr: false, + }, + { + name: "mixed HTTP and HTTPS", + containers: []api.Container{ + newContainer("10.210.0.2", + "app.example.com:8080/http", + "web.example.com:8000/http"), + newContainer("10.210.0.3", + "app.example.com:8080/http", + "secure.example.com:8888/https"), + newContainer("10.210.0.4", + "web.example.com:8000/http", + "secure.example.com:8888/https"), + newContainer("10.210.0.5", + "app.example.com:8080/http", + "web.example.com:8000/http", + "secure.example.com:8888/https"), + }, + want: `{ + "servers": { + "http": { + "listen": [":80"], + "routes": [ + { + "match": [{"host": ["app.example.com"]}], + "handle": [{ + "handler": "reverse_proxy", + "upstreams": [ + {"dial": "10.210.0.2:8080"}, + {"dial": "10.210.0.3:8080"}, + {"dial": "10.210.0.5:8080"} + ] + }] + }, + { + "match": [{"host": ["web.example.com"]}], + "handle": [{ + "handler": "reverse_proxy", + "upstreams": [ + {"dial": "10.210.0.2:8000"}, + {"dial": "10.210.0.4:8000"}, + {"dial": "10.210.0.5:8000"} + ] + }] + }, + { + "match": [{"path": ["/.uncloud-verify"]}], + "handle": [{ + "body": "verification-response-body", + "handler": "static_response", + "status_code": 200 + }] + } + ] + }, + "https": { + "listen": [":443"], + "routes": [ + { + "match": [{"host": ["secure.example.com"]}], + "handle": [{ + "handler": "reverse_proxy", + "upstreams": [ + {"dial": "10.210.0.3:8888"}, + {"dial": "10.210.0.4:8888"}, + {"dial": "10.210.0.5:8888"} + ] + }] + } + ] + } + } + }`, + wantErr: false, + }, + { + name: "container without uncloud network ignored", + containers: []api.Container{ + newContainerWithoutNetwork("ignored.example.com:8080/http"), + }, + want: `{ + "servers": { + "http": { + "listen": [":80"], + "routes": [{ + "match": [{"path": ["/.uncloud-verify"]}], + "handle": [{ + "body": "verification-response-body", + "handler": "static_response", + "status_code": 200 + }] + }] + }, + "https": { + "listen": [":443"] + } + } + }`, + wantErr: false, + }, + { + name: "container with invalid port ignored", + containers: []api.Container{ + newContainer("10.210.0.2", "invalid-port"), + }, + want: `{ + "servers": { + "http": { + "listen": [":80"], + "routes": [{ + "match": [{"path": ["/.uncloud-verify"]}], + "handle": [{ + "body": "verification-response-body", + "handler": "static_response", + "status_code": 200 + }] + }] + }, + "https": { + "listen": [":443"] + } + } + }`, + wantErr: false, + }, + { + name: "containers with unsupported protocols ignored", + containers: []api.Container{ + newContainer("10.210.0.2", "5000/tcp"), + newContainer("10.210.0.3", "5000/udp"), + newContainer("10.210.0.4", "80:8080/tcp@host"), + }, + want: `{ + "servers": { + "http": { + "listen": [":80"], + "routes": [{ + "match": [{"path": ["/.uncloud-verify"]}], + "handle": [{ + "body": "verification-response-body", + "handler": "static_response", + "status_code": 200 + }] + }] + }, + "https": { + "listen": [":443"] + } + } + }`, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config, err := GenerateConfig(tt.containers, "verification-response-body") + + if tt.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + + require.Len(t, config.AppsRaw, 1, "Expected one http app") + require.Contains(t, config.AppsRaw, "http", "Expected http app") + + assert.JSONEq(t, tt.want, string(config.AppsRaw["http"]), "Generated Caddy app config doesn't match") + }) + } +} + +func newContainer(ip string, ports ...string) api.Container { + portsLabel := strings.Join(ports, ",") + return api.Container{ + ContainerJSON: types.ContainerJSON{ + ContainerJSONBase: &types.ContainerJSONBase{}, + NetworkSettings: &types.NetworkSettings{ + Networks: map[string]*network.EndpointSettings{ + docker.NetworkName: { + IPAddress: ip, + }, + }, + }, + Config: &container.Config{ + Labels: map[string]string{ + api.LabelServicePorts: portsLabel, + }, + }, + }, + } +} + +func newContainerWithoutNetwork(ports ...string) api.Container { + portsLabel := strings.Join(ports, ",") + return api.Container{ + ContainerJSON: types.ContainerJSON{ + ContainerJSONBase: &types.ContainerJSONBase{}, + NetworkSettings: &types.NetworkSettings{ + Networks: map[string]*network.EndpointSettings{ + "other-network": { + IPAddress: "172.17.0.2", + }, + }, + }, + Config: &container.Config{ + Labels: map[string]string{ + api.LabelServicePorts: portsLabel, + }, + }, + }, + } +} diff --git a/internal/machine/caddyfile/controller.go b/internal/machine/caddyfile/controller.go index 0a549e1b..61045fce 100644 --- a/internal/machine/caddyfile/controller.go +++ b/internal/machine/caddyfile/controller.go @@ -3,21 +3,12 @@ package caddyfile import ( "context" "encoding/json" - "errors" "fmt" - "github.com/caddyserver/caddy/v2" - "github.com/caddyserver/caddy/v2/caddyconfig" - "github.com/caddyserver/caddy/v2/modules/caddyhttp" - "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy" "log/slog" - "net" - "net/http" "os" "path/filepath" - "strconv" "uncloud/internal/api" "uncloud/internal/fs" - "uncloud/internal/machine/docker" "uncloud/internal/machine/store" ) @@ -80,6 +71,9 @@ func (c *Controller) Run(ctx context.Context) error { continue } containers, err = c.filterAvailableContainers(containerRecords) + for _, c := range containers { + fmt.Printf("## caddyfile container: %+v\n", c) + } if err != nil { slog.Error("Failed to filter available containers.", "err", err) continue @@ -107,76 +101,9 @@ func (c *Controller) filterAvailableContainers(containerRecords []store.Containe } func (c *Controller) generateConfig(containers []api.Container) error { - // Maps hostnames to lists of upstreams (container IP:port pairs). - httpHostUpstreams := make(map[string][]string) - httpsHostUpstreams := make(map[string][]string) - for _, ctr := range containers { - logger := slog.With("container", ctr.ID) - network, ok := ctr.NetworkSettings.Networks[docker.NetworkName] - if !ok { - // Container is not connected to the uncloud Docker network (could be host network). - continue - } - if network.IPAddress == "" { - logger.Error("Container has no IPv4 address.") - continue - } - - ports, err := ctr.ServicePorts() - if err != nil { - logger.Error("Failed to parse service ports for container.", "err", err) - continue - } - - for _, port := range ports { - switch port.Protocol { - case api.ProtocolHTTP: - upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort))) - httpHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream) - case api.ProtocolHTTPS: - upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort))) - httpsHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream) - default: - if port.Mode == api.PortModeIngress { - // TODO: implement L4 ingress routing for TCP and UDP. - logger.Error("Unsupported protocol for ingress port.", "port", port) - continue - } - } - } - } - - var warnings []caddyconfig.Warning - servers := make(map[string]*caddyhttp.Server) - servers["http"] = &caddyhttp.Server{ - Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)}, - Routes: append( - hostUpstreamsToRoutes(httpHostUpstreams, &warnings), - // Add a route to respond with a static verification response at the /.uncloud-verify path. - verificationRoute(c.verifyResponse, &warnings), - ), - } - servers["https"] = &caddyhttp.Server{ - Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)}, - Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings), - } - - httpApp := caddyhttp.App{ - Servers: servers, - } - config := &caddy.Config{ - AppsRaw: caddy.ModuleMap{ - "http": caddyconfig.JSON(httpApp, &warnings), - }, - } - - var err error - if len(warnings) > 0 { - // warnings only contains errors from JSON marshaling, which are highly unlikely with correct code. - for _, w := range warnings { - err = errors.Join(err, errors.New(w.Message)) - } - return fmt.Errorf("marshal Caddy configuration: %w", err) + config, err := GenerateConfig(containers, c.verifyResponse) + if err != nil { + return err } configBytes, err := json.MarshalIndent(config, "", " ") @@ -193,68 +120,3 @@ func (c *Controller) generateConfig(containers []api.Container) error { return nil } - -// hostUpstreamsToRoutes converts a map of hostnames to upstreams to a list of Caddy routes. -func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyconfig.Warning) []caddyhttp.Route { - routes := make([]caddyhttp.Route, 0, len(hostUpstreams)) - for hostname, upstreams := range hostUpstreams { - upstreamPool := make([]*reverseproxy.Upstream, len(upstreams)) - for i, upstream := range upstreams { - upstreamPool[i] = &reverseproxy.Upstream{ - Dial: upstream, - } - } - handler := &reverseproxy.Handler{ - Upstreams: upstreamPool, - } - - routes = append(routes, caddyhttp.Route{ - MatcherSetsRaw: caddyhttp.RawMatcherSets{ - { - "host": caddyconfig.JSON(caddyhttp.MatchHost{hostname}, warnings), - }, - }, - HandlersRaw: []json.RawMessage{ - caddyconfig.JSONModuleObject(handler, "handler", "reverse_proxy", warnings), - }, - }) - } - return routes -} - -// verificationRoute returns a Caddy route that responds with the given static response at the /.uncloud-verify path. -func verificationRoute(response string, warnings *[]caddyconfig.Warning) caddyhttp.Route { - // Return the following route: - // { - // "match": [ - // { - // "path": [ - // "/.uncloud-verify" - // ] - // } - // ], - // "handle": [ - // { - // "handler": "static_response", - // "body": "", - // "status_code": 200 - // } - // ] - // } - - staticResponse := caddyhttp.StaticResponse{ - StatusCode: caddyhttp.WeakString(strconv.Itoa(http.StatusOK)), - Body: response, - } - - return caddyhttp.Route{ - MatcherSetsRaw: caddyhttp.RawMatcherSets{ - { - "path": caddyconfig.JSON(caddyhttp.MatchPath{VerifyPath}, warnings), - }, - }, - HandlersRaw: []json.RawMessage{ - caddyconfig.JSONModuleObject(staticResponse, "handler", "static_response", warnings), - }, - } -}