fix(caddy): config generation for multiple https containers

This commit is contained in:
Pavel Sviderski
2025-03-03 15:04:04 +10:00
parent 3f4f263c05
commit d9e350c4dd
3 changed files with 534 additions and 144 deletions
+163
View File
@@ -0,0 +1,163 @@
package caddyfile
import (
"encoding/json"
"errors"
"fmt"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
"log/slog"
"maps"
"net"
"net/http"
"slices"
"strconv"
"uncloud/internal/api"
"uncloud/internal/machine/docker"
)
func GenerateConfig(containers []api.Container, verifyResponse string) (*caddy.Config, error) {
// Maps hostnames to lists of upstreams (container IP:port pairs).
httpHostUpstreams := make(map[string][]string)
httpsHostUpstreams := make(map[string][]string)
for _, ctr := range containers {
logger := slog.With("container", ctr.ID)
network, ok := ctr.NetworkSettings.Networks[docker.NetworkName]
if !ok {
// Container is not connected to the uncloud Docker network (could be host network).
continue
}
if network.IPAddress == "" {
logger.Error("Container has no IPv4 address.")
continue
}
ports, err := ctr.ServicePorts()
if err != nil {
logger.Error("Failed to parse service ports for container.", "err", err)
continue
}
for _, port := range ports {
switch port.Protocol {
case api.ProtocolHTTP:
upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort)))
httpHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream)
case api.ProtocolHTTPS:
upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort)))
httpsHostUpstreams[port.Hostname] = append(httpsHostUpstreams[port.Hostname], upstream)
default:
// TODO: implement L4 ingress routing for TCP and UDP.
logger.Error("Unsupported protocol for ingress port.", "port", port)
continue
}
}
}
var warnings []caddyconfig.Warning
servers := make(map[string]*caddyhttp.Server)
servers["http"] = &caddyhttp.Server{
Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)},
Routes: append(
hostUpstreamsToRoutes(httpHostUpstreams, &warnings),
// Add a route to respond with a static verification response at the /.uncloud-verify path.
verificationRoute(verifyResponse, &warnings),
),
}
servers["https"] = &caddyhttp.Server{
Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)},
Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings),
}
httpApp := caddyhttp.App{
Servers: servers,
}
config := &caddy.Config{
AppsRaw: caddy.ModuleMap{
"http": caddyconfig.JSON(httpApp, &warnings),
},
}
var err error
if len(warnings) > 0 {
// warnings only contains errors from JSON marshaling, which are highly unlikely with correct code.
for _, w := range warnings {
err = errors.Join(err, errors.New(w.Message))
}
return nil, fmt.Errorf("marshal Caddy configuration: %w", err)
}
return config, nil
}
// hostUpstreamsToRoutes converts a map of hostnames to upstreams to a list of Caddy routes.
func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyconfig.Warning) []caddyhttp.Route {
// Sort hostnames for deterministic output.
hostnames := slices.Collect(maps.Keys(hostUpstreams))
slices.Sort(hostnames)
routes := make([]caddyhttp.Route, 0, len(hostUpstreams))
for _, hostname := range hostnames {
upstreams := hostUpstreams[hostname]
upstreamPool := make([]*reverseproxy.Upstream, len(upstreams))
for i, upstream := range upstreams {
upstreamPool[i] = &reverseproxy.Upstream{
Dial: upstream,
}
}
handler := &reverseproxy.Handler{
Upstreams: upstreamPool,
}
routes = append(routes, caddyhttp.Route{
MatcherSetsRaw: caddyhttp.RawMatcherSets{
{
"host": caddyconfig.JSON(caddyhttp.MatchHost{hostname}, warnings),
},
},
HandlersRaw: []json.RawMessage{
caddyconfig.JSONModuleObject(handler, "handler", "reverse_proxy", warnings),
},
})
}
return routes
}
// verificationRoute returns a Caddy route that responds with the given static response at the /.uncloud-verify path.
func verificationRoute(response string, warnings *[]caddyconfig.Warning) caddyhttp.Route {
// Return the following route:
// {
// "match": [
// {
// "path": [
// "/.uncloud-verify"
// ]
// }
// ],
// "handle": [
// {
// "handler": "static_response",
// "body": "<response>",
// "status_code": 200
// }
// ]
// }
staticResponse := caddyhttp.StaticResponse{
StatusCode: caddyhttp.WeakString(strconv.Itoa(http.StatusOK)),
Body: response,
}
return caddyhttp.Route{
MatcherSetsRaw: caddyhttp.RawMatcherSets{
{
"path": caddyconfig.JSON(caddyhttp.MatchPath{VerifyPath}, warnings),
},
},
HandlersRaw: []json.RawMessage{
caddyconfig.JSONModuleObject(staticResponse, "handler", "static_response", warnings),
},
}
}
+365
View File
@@ -0,0 +1,365 @@
package caddyfile
import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/network"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"strings"
"testing"
"uncloud/internal/api"
"uncloud/internal/machine/docker"
)
func TestGenerateConfig(t *testing.T) {
tests := []struct {
name string
containers []api.Container
want string
wantErr bool
}{
{
name: "empty containers",
containers: []api.Container{},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}]
},
"https": {
"listen": [":443"]
}
}
}`,
wantErr: false,
},
{
name: "HTTP container",
containers: []api.Container{
newContainer("10.210.0.2", "app.example.com:8080/http"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [
{
"match": [{"host": ["app.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"upstreams": [{"dial": "10.210.0.2:8080"}]
}]
},
{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}
]
},
"https": {
"listen": [":443"]
}
}
}`,
wantErr: false,
},
{
name: "load balancing multiple containers",
containers: []api.Container{
newContainer("10.210.0.2", "app.example.com:8080/http"),
newContainer("10.210.0.3", "app.example.com:8080/http"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [
{
"match": [{"host": ["app.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"upstreams": [
{"dial": "10.210.0.2:8080"},
{"dial": "10.210.0.3:8080"}
]
}]
},
{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}
]
},
"https": {
"listen": [":443"]
}
}
}`,
wantErr: false,
},
{
name: "HTTPS container",
containers: []api.Container{
newContainer("10.210.0.2", "secure.example.com:8000/https"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [
{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}
]
},
"https": {
"listen": [":443"],
"routes": [
{
"match": [{"host": ["secure.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"upstreams": [{"dial": "10.210.0.2:8000"}]
}]
}
]
}
}
}`,
wantErr: false,
},
{
name: "mixed HTTP and HTTPS",
containers: []api.Container{
newContainer("10.210.0.2",
"app.example.com:8080/http",
"web.example.com:8000/http"),
newContainer("10.210.0.3",
"app.example.com:8080/http",
"secure.example.com:8888/https"),
newContainer("10.210.0.4",
"web.example.com:8000/http",
"secure.example.com:8888/https"),
newContainer("10.210.0.5",
"app.example.com:8080/http",
"web.example.com:8000/http",
"secure.example.com:8888/https"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [
{
"match": [{"host": ["app.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"upstreams": [
{"dial": "10.210.0.2:8080"},
{"dial": "10.210.0.3:8080"},
{"dial": "10.210.0.5:8080"}
]
}]
},
{
"match": [{"host": ["web.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"upstreams": [
{"dial": "10.210.0.2:8000"},
{"dial": "10.210.0.4:8000"},
{"dial": "10.210.0.5:8000"}
]
}]
},
{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}
]
},
"https": {
"listen": [":443"],
"routes": [
{
"match": [{"host": ["secure.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"upstreams": [
{"dial": "10.210.0.3:8888"},
{"dial": "10.210.0.4:8888"},
{"dial": "10.210.0.5:8888"}
]
}]
}
]
}
}
}`,
wantErr: false,
},
{
name: "container without uncloud network ignored",
containers: []api.Container{
newContainerWithoutNetwork("ignored.example.com:8080/http"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}]
},
"https": {
"listen": [":443"]
}
}
}`,
wantErr: false,
},
{
name: "container with invalid port ignored",
containers: []api.Container{
newContainer("10.210.0.2", "invalid-port"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}]
},
"https": {
"listen": [":443"]
}
}
}`,
wantErr: false,
},
{
name: "containers with unsupported protocols ignored",
containers: []api.Container{
newContainer("10.210.0.2", "5000/tcp"),
newContainer("10.210.0.3", "5000/udp"),
newContainer("10.210.0.4", "80:8080/tcp@host"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}]
},
"https": {
"listen": [":443"]
}
}
}`,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config, err := GenerateConfig(tt.containers, "verification-response-body")
if tt.wantErr {
assert.Error(t, err)
return
}
require.NoError(t, err)
require.Len(t, config.AppsRaw, 1, "Expected one http app")
require.Contains(t, config.AppsRaw, "http", "Expected http app")
assert.JSONEq(t, tt.want, string(config.AppsRaw["http"]), "Generated Caddy app config doesn't match")
})
}
}
func newContainer(ip string, ports ...string) api.Container {
portsLabel := strings.Join(ports, ",")
return api.Container{
ContainerJSON: types.ContainerJSON{
ContainerJSONBase: &types.ContainerJSONBase{},
NetworkSettings: &types.NetworkSettings{
Networks: map[string]*network.EndpointSettings{
docker.NetworkName: {
IPAddress: ip,
},
},
},
Config: &container.Config{
Labels: map[string]string{
api.LabelServicePorts: portsLabel,
},
},
},
}
}
func newContainerWithoutNetwork(ports ...string) api.Container {
portsLabel := strings.Join(ports, ",")
return api.Container{
ContainerJSON: types.ContainerJSON{
ContainerJSONBase: &types.ContainerJSONBase{},
NetworkSettings: &types.NetworkSettings{
Networks: map[string]*network.EndpointSettings{
"other-network": {
IPAddress: "172.17.0.2",
},
},
},
Config: &container.Config{
Labels: map[string]string{
api.LabelServicePorts: portsLabel,
},
},
},
}
}
+6 -144
View File
@@ -3,21 +3,12 @@ package caddyfile
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
"log/slog"
"net"
"net/http"
"os"
"path/filepath"
"strconv"
"uncloud/internal/api"
"uncloud/internal/fs"
"uncloud/internal/machine/docker"
"uncloud/internal/machine/store"
)
@@ -80,6 +71,9 @@ func (c *Controller) Run(ctx context.Context) error {
continue
}
containers, err = c.filterAvailableContainers(containerRecords)
for _, c := range containers {
fmt.Printf("## caddyfile container: %+v\n", c)
}
if err != nil {
slog.Error("Failed to filter available containers.", "err", err)
continue
@@ -107,76 +101,9 @@ func (c *Controller) filterAvailableContainers(containerRecords []store.Containe
}
func (c *Controller) generateConfig(containers []api.Container) error {
// Maps hostnames to lists of upstreams (container IP:port pairs).
httpHostUpstreams := make(map[string][]string)
httpsHostUpstreams := make(map[string][]string)
for _, ctr := range containers {
logger := slog.With("container", ctr.ID)
network, ok := ctr.NetworkSettings.Networks[docker.NetworkName]
if !ok {
// Container is not connected to the uncloud Docker network (could be host network).
continue
}
if network.IPAddress == "" {
logger.Error("Container has no IPv4 address.")
continue
}
ports, err := ctr.ServicePorts()
if err != nil {
logger.Error("Failed to parse service ports for container.", "err", err)
continue
}
for _, port := range ports {
switch port.Protocol {
case api.ProtocolHTTP:
upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort)))
httpHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream)
case api.ProtocolHTTPS:
upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort)))
httpsHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream)
default:
if port.Mode == api.PortModeIngress {
// TODO: implement L4 ingress routing for TCP and UDP.
logger.Error("Unsupported protocol for ingress port.", "port", port)
continue
}
}
}
}
var warnings []caddyconfig.Warning
servers := make(map[string]*caddyhttp.Server)
servers["http"] = &caddyhttp.Server{
Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)},
Routes: append(
hostUpstreamsToRoutes(httpHostUpstreams, &warnings),
// Add a route to respond with a static verification response at the /.uncloud-verify path.
verificationRoute(c.verifyResponse, &warnings),
),
}
servers["https"] = &caddyhttp.Server{
Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)},
Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings),
}
httpApp := caddyhttp.App{
Servers: servers,
}
config := &caddy.Config{
AppsRaw: caddy.ModuleMap{
"http": caddyconfig.JSON(httpApp, &warnings),
},
}
var err error
if len(warnings) > 0 {
// warnings only contains errors from JSON marshaling, which are highly unlikely with correct code.
for _, w := range warnings {
err = errors.Join(err, errors.New(w.Message))
}
return fmt.Errorf("marshal Caddy configuration: %w", err)
config, err := GenerateConfig(containers, c.verifyResponse)
if err != nil {
return err
}
configBytes, err := json.MarshalIndent(config, "", " ")
@@ -193,68 +120,3 @@ func (c *Controller) generateConfig(containers []api.Container) error {
return nil
}
// hostUpstreamsToRoutes converts a map of hostnames to upstreams to a list of Caddy routes.
func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyconfig.Warning) []caddyhttp.Route {
routes := make([]caddyhttp.Route, 0, len(hostUpstreams))
for hostname, upstreams := range hostUpstreams {
upstreamPool := make([]*reverseproxy.Upstream, len(upstreams))
for i, upstream := range upstreams {
upstreamPool[i] = &reverseproxy.Upstream{
Dial: upstream,
}
}
handler := &reverseproxy.Handler{
Upstreams: upstreamPool,
}
routes = append(routes, caddyhttp.Route{
MatcherSetsRaw: caddyhttp.RawMatcherSets{
{
"host": caddyconfig.JSON(caddyhttp.MatchHost{hostname}, warnings),
},
},
HandlersRaw: []json.RawMessage{
caddyconfig.JSONModuleObject(handler, "handler", "reverse_proxy", warnings),
},
})
}
return routes
}
// verificationRoute returns a Caddy route that responds with the given static response at the /.uncloud-verify path.
func verificationRoute(response string, warnings *[]caddyconfig.Warning) caddyhttp.Route {
// Return the following route:
// {
// "match": [
// {
// "path": [
// "/.uncloud-verify"
// ]
// }
// ],
// "handle": [
// {
// "handler": "static_response",
// "body": "<response>",
// "status_code": 200
// }
// ]
// }
staticResponse := caddyhttp.StaticResponse{
StatusCode: caddyhttp.WeakString(strconv.Itoa(http.StatusOK)),
Body: response,
}
return caddyhttp.Route{
MatcherSetsRaw: caddyhttp.RawMatcherSets{
{
"path": caddyconfig.JSON(caddyhttp.MatchPath{VerifyPath}, warnings),
},
},
HandlersRaw: []json.RawMessage{
caddyconfig.JSONModuleObject(staticResponse, "handler", "static_response", warnings),
},
}
}