From 38800924edf19f20142d9d958ca23686832ea20b Mon Sep 17 00:00:00 2001 From: Pavel Sviderski Date: Thu, 1 May 2025 11:47:05 +1000 Subject: [PATCH] fix(caddy): proxy to only healthy containers --- internal/machine/caddyfile/config.go | 29 ++-- internal/machine/caddyfile/config_test.go | 155 +++++++++++++--------- 2 files changed, 105 insertions(+), 79 deletions(-) diff --git a/internal/machine/caddyfile/config.go b/internal/machine/caddyfile/config.go index d73e7645..31fcc0ba 100644 --- a/internal/machine/caddyfile/config.go +++ b/internal/machine/caddyfile/config.go @@ -15,7 +15,6 @@ import ( "github.com/caddyserver/caddy/v2/caddyconfig" "github.com/caddyserver/caddy/v2/modules/caddyhttp" "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy" - "github.com/psviderski/uncloud/internal/machine/docker" "github.com/psviderski/uncloud/pkg/api" ) @@ -24,34 +23,38 @@ func GenerateConfig(containers []api.ServiceContainer, verifyResponse string) (* httpHostUpstreams := make(map[string][]string) httpsHostUpstreams := make(map[string][]string) for _, ctr := range containers { - logger := slog.With("container", ctr.ID) - network, ok := ctr.NetworkSettings.Networks[docker.NetworkName] - if !ok { - // Container is not connected to the uncloud Docker network (could be host network). - continue - } - if network.IPAddress == "" { - logger.Error("Container has no IPv4 address.") + if !ctr.Healthy() { continue } + ip := ctr.UncloudNetworkIP() + if !ip.IsValid() { + // Container is not connected to the uncloud Docker network (could be host network). + continue + } + log := slog.With("container", ctr.ID) + ports, err := ctr.ServicePorts() if err != nil { - logger.Error("Failed to parse service ports for container.", "err", err) + log.Error("Failed to parse service ports for container.", "err", err) continue } for _, port := range ports { + if port.Mode != api.PortModeIngress { + continue + } + switch port.Protocol { case api.ProtocolHTTP: - upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort))) + upstream := net.JoinHostPort(ip.String(), strconv.Itoa(int(port.ContainerPort))) httpHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream) case api.ProtocolHTTPS: - upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort))) + upstream := net.JoinHostPort(ip.String(), strconv.Itoa(int(port.ContainerPort))) httpsHostUpstreams[port.Hostname] = append(httpsHostUpstreams[port.Hostname], upstream) default: // TODO: implement L4 ingress routing for TCP and UDP. - logger.Error("Unsupported protocol for ingress port.", "port", port) + log.Error("Unsupported protocol for ingress port.", "port", port) continue } } diff --git a/internal/machine/caddyfile/config_test.go b/internal/machine/caddyfile/config_test.go index 273c3a5f..fa865ea7 100644 --- a/internal/machine/caddyfile/config_test.go +++ b/internal/machine/caddyfile/config_test.go @@ -14,6 +14,25 @@ import ( ) func TestGenerateConfig(t *testing.T) { + configWithoutServices := `{ + "servers": { + "http": { + "listen": [":80"], + "routes": [{ + "match": [{"path": ["/.uncloud-verify"]}], + "handle": [{ + "body": "verification-response-body", + "handler": "static_response", + "status_code": 200 + }] + }] + }, + "https": { + "listen": [":443"] + } + } + }` + tests := []struct { name string containers []api.ServiceContainer @@ -23,25 +42,8 @@ func TestGenerateConfig(t *testing.T) { { name: "empty containers", containers: []api.ServiceContainer{}, - want: `{ - "servers": { - "http": { - "listen": [":80"], - "routes": [{ - "match": [{"path": ["/.uncloud-verify"]}], - "handle": [{ - "body": "verification-response-body", - "handler": "static_response", - "status_code": 200 - }] - }] - }, - "https": { - "listen": [":443"] - } - } - }`, - wantErr: false, + want: configWithoutServices, + wantErr: false, }, { @@ -231,24 +233,7 @@ func TestGenerateConfig(t *testing.T) { containers: []api.ServiceContainer{ newContainerWithoutNetwork("ignored.example.com:8080/http"), }, - want: `{ - "servers": { - "http": { - "listen": [":80"], - "routes": [{ - "match": [{"path": ["/.uncloud-verify"]}], - "handle": [{ - "body": "verification-response-body", - "handler": "static_response", - "status_code": 200 - }] - }] - }, - "https": { - "listen": [":443"] - } - } - }`, + want: configWithoutServices, wantErr: false, }, { @@ -256,45 +241,63 @@ func TestGenerateConfig(t *testing.T) { containers: []api.ServiceContainer{ newContainer("10.210.0.2", "invalid-port"), }, - want: `{ - "servers": { - "http": { - "listen": [":80"], - "routes": [{ - "match": [{"path": ["/.uncloud-verify"]}], - "handle": [{ - "body": "verification-response-body", - "handler": "static_response", - "status_code": 200 - }] - }] - }, - "https": { - "listen": [":443"] - } - } - }`, + want: configWithoutServices, wantErr: false, }, { - name: "containers with unsupported protocols ignored", + name: "containers with unsupported protocols and host mode ignored", containers: []api.ServiceContainer{ newContainer("10.210.0.2", "5000/tcp"), newContainer("10.210.0.3", "5000/udp"), newContainer("10.210.0.4", "80:8080/tcp@host"), }, + want: configWithoutServices, + wantErr: false, + }, + { + name: "restarting container ignored", + containers: []api.ServiceContainer{ + newRestartingContainer("10.210.0.2", "app.example.com:8080/http"), + }, + want: configWithoutServices, + wantErr: false, + }, + { + name: "stopped container ignored", + containers: []api.ServiceContainer{ + newStoppedContainer("10.210.0.2", "app.example.com:8080/http"), + }, + want: configWithoutServices, + wantErr: false, + }, + { + name: "mix of running, restarting, and stopped containers", + containers: []api.ServiceContainer{ + newContainer("10.210.0.2", "app.example.com:8080/http"), + newRestartingContainer("10.210.0.3", "app.example.com:8080/http"), + newStoppedContainer("10.210.0.4", "app.example.com:8080/http"), + }, want: `{ "servers": { "http": { "listen": [":80"], - "routes": [{ - "match": [{"path": ["/.uncloud-verify"]}], - "handle": [{ - "body": "verification-response-body", - "handler": "static_response", - "status_code": 200 - }] - }] + "routes": [ + { + "match": [{"host": ["app.example.com"]}], + "handle": [{ + "handler": "reverse_proxy", + "upstreams": [{"dial": "10.210.0.2:8080"}] + }] + }, + { + "match": [{"path": ["/.uncloud-verify"]}], + "handle": [{ + "body": "verification-response-body", + "handler": "static_response", + "status_code": 200 + }] + } + ] }, "https": { "listen": [":443"] @@ -326,7 +329,11 @@ func TestGenerateConfig(t *testing.T) { func newContainer(ip string, ports ...string) api.ServiceContainer { portsLabel := strings.Join(ports, ",") return api.ServiceContainer{Container: api.Container{ContainerJSON: types.ContainerJSON{ - ContainerJSONBase: &types.ContainerJSONBase{}, + ContainerJSONBase: &types.ContainerJSONBase{ + State: &types.ContainerState{ + Running: true, + }, + }, NetworkSettings: &types.NetworkSettings{ Networks: map[string]*network.EndpointSettings{ docker.NetworkName: { @@ -345,7 +352,11 @@ func newContainer(ip string, ports ...string) api.ServiceContainer { func newContainerWithoutNetwork(ports ...string) api.ServiceContainer { portsLabel := strings.Join(ports, ",") return api.ServiceContainer{Container: api.Container{ContainerJSON: types.ContainerJSON{ - ContainerJSONBase: &types.ContainerJSONBase{}, + ContainerJSONBase: &types.ContainerJSONBase{ + State: &types.ContainerState{ + Running: true, + }, + }, NetworkSettings: &types.NetworkSettings{ Networks: map[string]*network.EndpointSettings{ "other-network": { @@ -360,3 +371,15 @@ func newContainerWithoutNetwork(ports ...string) api.ServiceContainer { }, }}} } + +func newRestartingContainer(ip string, ports ...string) api.ServiceContainer { + ctr := newContainer(ip, ports...) + ctr.Container.State.Restarting = true + return ctr +} + +func newStoppedContainer(ip string, ports ...string) api.ServiceContainer { + ctr := newContainer(ip, ports...) + ctr.Container.State.Running = false + return ctr +}