From 2b3f325075118edeabfec897c14d08ccb96434c7 Mon Sep 17 00:00:00 2001 From: Pavel Sviderski Date: Mon, 23 Dec 2024 20:41:04 +1000 Subject: [PATCH] generate caddy config with routes from published http(s) ports --- go.mod | 1 + go.sum | 6 +- internal/machine/caddyfile/controller.go | 87 ++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 9a4a7418..d69e68d3 100644 --- a/go.mod +++ b/go.mod @@ -221,6 +221,7 @@ require ( github.com/onsi/ginkgo/v2 v2.22.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pires/go-proxyproto v0.7.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 04dcd755..2e4d31f2 100644 --- a/go.sum +++ b/go.sum @@ -864,6 +864,8 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/webrtc/v3 v3.2.42 h1:WN/ZuMjtpQOoGRCZUg/zFG+JHEvYLVyDKOxU6H1qWlE= github.com/pion/webrtc/v3 v3.2.42/go.mod h1:M1RAe3TNTD1tzyvqHrbVODfwdPGSXOUo/OgpoGGJqFY= +github.com/pires/go-proxyproto v0.7.0 h1:IukmRewDQFWC7kfnb66CSomk2q/seBuilHBYFwyq0Hs= +github.com/pires/go-proxyproto v0.7.0/go.mod h1:Vz/1JPY/OACxWGQNIRY2BeyDmpoaWmEP40O9LbuiFR4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1136,8 +1138,6 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap/exp v0.2.0 h1:FtGenNNeCATRB3CmB/yEUnjEFeJWpB/pMcy7e2bKPYs= go.uber.org/zap/exp v0.2.0/go.mod h1:t0gqAIdh1MfKv9EwN/dLwfZnJxe9ITAZN78HEWPFWDQ= -go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= -go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1372,6 +1372,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1 h1:d4KQkxAaAiRY2h5Zqis161Pv91A37uZyJOx73duwUwM= gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1/go.mod h1:WbjuEoo1oadwzQ4apSDU+JTvmllEHtsNHS6y7vFc7iw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/internal/machine/caddyfile/controller.go b/internal/machine/caddyfile/controller.go index a4e2f530..f3aadbbc 100644 --- a/internal/machine/caddyfile/controller.go +++ b/internal/machine/caddyfile/controller.go @@ -3,15 +3,20 @@ package caddyfile import ( "context" "encoding/json" + "errors" "fmt" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig" "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy" "log/slog" + "net" "os" "path/filepath" + "strconv" "uncloud/internal/api" "uncloud/internal/fs" + "uncloud/internal/machine/docker" "uncloud/internal/machine/store" ) @@ -96,28 +101,72 @@ func (c *Controller) filterAvailableContainers(containerRecords []*store.Contain } func (c *Controller) generateConfig(containers []*api.Container) error { + // Maps hostnames to lists of upstreams (container IP:port pairs). + httpHostUpstreams := make(map[string][]string) + httpsHostUpstreams := make(map[string][]string) + for _, ctr := range containers { + logger := slog.With("container", ctr.ID) + network, ok := ctr.NetworkSettings.Networks[docker.NetworkName] + if !ok { + // Container is not connected to the uncloud Docker network (could be host network). + continue + } + if network.IPAddress == "" { + logger.Error("Container has no IPv4 address.") + continue + } + + ports, err := ctr.ServicePorts() + if err != nil { + logger.Error("Failed to parse service ports for container.", "err", err) + continue + } + + for _, port := range ports { + switch port.Protocol { + case api.ProtocolHTTP: + upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort))) + httpHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream) + case api.ProtocolHTTPS: + upstream := net.JoinHostPort(network.IPAddress, strconv.Itoa(int(port.ContainerPort))) + httpsHostUpstreams[port.Hostname] = append(httpHostUpstreams[port.Hostname], upstream) + default: + if port.Mode == api.PortModeIngress { + // TODO: implement L4 ingress routing for TCP and UDP. + logger.Error("Unsupported protocol for ingress port.", "port", port) + continue + } + } + } + } + + var warnings []caddyconfig.Warning servers := make(map[string]*caddyhttp.Server) servers["http"] = &caddyhttp.Server{ Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)}, + Routes: hostUpstreamsToRoutes(httpHostUpstreams, &warnings), } servers["https"] = &caddyhttp.Server{ Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)}, + Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings), } httpApp := caddyhttp.App{ Servers: servers, } - config := &caddy.Config{ - AppsRaw: make(caddy.ModuleMap), + AppsRaw: caddy.ModuleMap{ + "http": caddyconfig.JSON(httpApp, &warnings), + }, } - var warnings []caddyconfig.Warning - config.AppsRaw["http"] = caddyconfig.JSON(httpApp, &warnings) + var err error if len(warnings) > 0 { + // warnings only contains errors from JSON marshaling, which are highly unlikely with correct code. for _, w := range warnings { - slog.Warn("Generate Caddy configuration warning.", "warn", w.String()) + err = errors.Join(err, errors.New(w.Message)) } + return fmt.Errorf("marshal Caddy configuration: %w", err) } configBytes, err := json.MarshalIndent(config, "", " ") @@ -134,3 +183,31 @@ func (c *Controller) generateConfig(containers []*api.Container) error { return nil } + +// hostUpstreamsToRoutes converts a map of hostnames to upstreams to a list of Caddy routes. +func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyconfig.Warning) []caddyhttp.Route { + routes := make([]caddyhttp.Route, 0, len(hostUpstreams)) + for hostname, upstreams := range hostUpstreams { + upstreamPool := make([]*reverseproxy.Upstream, len(upstreams)) + for i, upstream := range upstreams { + upstreamPool[i] = &reverseproxy.Upstream{ + Dial: upstream, + } + } + handler := &reverseproxy.Handler{ + Upstreams: upstreamPool, + } + + routes = append(routes, caddyhttp.Route{ + MatcherSetsRaw: caddyhttp.RawMatcherSets{ + { + "host": caddyconfig.JSON(caddyhttp.MatchHost{hostname}, warnings), + }, + }, + HandlersRaw: []json.RawMessage{ + caddyconfig.JSONModuleObject(handler, "handler", "reverse_proxy", warnings), + }, + }) + } + return routes +}