philipp/uncloud
1
0
Fork 0
mirror of https://github.com/psviderski/uncloud.git synced 2026-08-26 19:13:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: enable passive health checks and upstream retries in Caddy

Browse Source
This commit is contained in:
Pavel Sviderski
2025-06-12 18:38:28 +10:00
parent 2a0d31cec0
commit fc61e2fc07
6 changed files with 106 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/machine/caddyfile/config.go → internal/machine/caddyconfig/config.go
+16 -1
View File
@@ -1,4 +1,4 @@
package caddyfile package caddyconfig
import ( import (
"encoding/json" "encoding/json"
@@ -10,6 +10,7 @@ import (
"net/http" "net/http"
"slices" "slices"
"strconv" "strconv"
"time"
"github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig" "github.com/caddyserver/caddy/v2/caddyconfig"
@@ -64,6 +65,8 @@ func GenerateConfig(containers []api.ServiceContainer, verifyResponse string) (*
servers := make(map[string]*caddyhttp.Server) servers := make(map[string]*caddyhttp.Server)
servers["http"] = &caddyhttp.Server{ servers["http"] = &caddyhttp.Server{
Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)}, Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)},
// All http requests to this server are logged to the default logger.
Logs: &caddyhttp.ServerLogConfig{},
Routes: append( Routes: append(
hostUpstreamsToRoutes(httpHostUpstreams, &warnings), hostUpstreamsToRoutes(httpHostUpstreams, &warnings),
// Add a route to respond with a static verification response at the /.uncloud-verify path. // Add a route to respond with a static verification response at the /.uncloud-verify path.
@@ -72,6 +75,8 @@ func GenerateConfig(containers []api.ServiceContainer, verifyResponse string) (*
} }
servers["https"] = &caddyhttp.Server{ servers["https"] = &caddyhttp.Server{
Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)}, Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)},
// All https requests to this server are logged to the default logger.
Logs: &caddyhttp.ServerLogConfig{},
Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings), Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings),
} }
@@ -112,6 +117,16 @@ func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyc
} }
} }
handler := &reverseproxy.Handler{ handler := &reverseproxy.Handler{
HealthChecks: &reverseproxy.HealthChecks{
// Enable passive health checks to automatically detect unhealthy upstreams.
Passive: &reverseproxy.PassiveHealthChecks{
FailDuration: caddy.Duration(30 * time.Second),
},
},
LoadBalancing: &reverseproxy.LoadBalancing{
// Retry failed requests to skip over temporarily unavailable upstreams.
Retries: 3,
},
Upstreams: upstreamPool, Upstreams: upstreamPool,
} }
internal/machine/caddyfile/config_test.go → internal/machine/caddyconfig/config_test.go
+81 -13
View File
@@ -1,4 +1,4 @@
package caddyfile package caddyconfig
import ( import (
"strings" "strings"
@@ -25,10 +25,12 @@ func TestGenerateConfig(t *testing.T) {
"handler": "static_response", "handler": "static_response",
"status_code": 200 "status_code": 200
}] }]
}] }],
"logs": {}
}, },
"https": { "https": {
"listen": [":443"] "listen": [":443"],
"logs": {}
} }
} }
}` }`
@@ -60,6 +62,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["app.example.com"]}], "match": [{"host": ["app.example.com"]}],
"handle": [{ "handle": [{
"handler": "reverse_proxy", "handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [{"dial": "10.210.0.2:8080"}] "upstreams": [{"dial": "10.210.0.2:8080"}]
}] }]
}, },
@@ -71,10 +81,12 @@ func TestGenerateConfig(t *testing.T) {
"status_code": 200 "status_code": 200
}] }]
} }
] ],
"logs": {}
}, },
"https": { "https": {
"listen": [":443"] "listen": [":443"],
"logs": {}
} }
} }
}`, }`,
@@ -95,6 +107,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["app.example.com"]}], "match": [{"host": ["app.example.com"]}],
"handle": [{ "handle": [{
"handler": "reverse_proxy", "handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [ "upstreams": [
{"dial": "10.210.0.2:8080"}, {"dial": "10.210.0.2:8080"},
{"dial": "10.210.0.3:8080"} {"dial": "10.210.0.3:8080"}
@@ -109,10 +129,12 @@ func TestGenerateConfig(t *testing.T) {
"status_code": 200 "status_code": 200
}] }]
} }
] ],
"logs": {}
}, },
"https": { "https": {
"listen": [":443"] "listen": [":443"],
"logs": {}
} }
} }
}`, }`,
@@ -136,7 +158,8 @@ func TestGenerateConfig(t *testing.T) {
"status_code": 200 "status_code": 200
}] }]
} }
] ],
"logs": {}
}, },
"https": { "https": {
"listen": [":443"], "listen": [":443"],
@@ -145,10 +168,19 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["secure.example.com"]}], "match": [{"host": ["secure.example.com"]}],
"handle": [{ "handle": [{
"handler": "reverse_proxy", "handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [{"dial": "10.210.0.2:8000"}] "upstreams": [{"dial": "10.210.0.2:8000"}]
}] }]
} }
] ],
"logs": {}
} }
} }
}`, }`,
@@ -180,6 +212,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["app.example.com"]}], "match": [{"host": ["app.example.com"]}],
"handle": [{ "handle": [{
"handler": "reverse_proxy", "handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [ "upstreams": [
{"dial": "10.210.0.2:8080"}, {"dial": "10.210.0.2:8080"},
{"dial": "10.210.0.3:8080"}, {"dial": "10.210.0.3:8080"},
@@ -191,6 +231,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["web.example.com"]}], "match": [{"host": ["web.example.com"]}],
"handle": [{ "handle": [{
"handler": "reverse_proxy", "handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [ "upstreams": [
{"dial": "10.210.0.2:8000"}, {"dial": "10.210.0.2:8000"},
{"dial": "10.210.0.4:8000"}, {"dial": "10.210.0.4:8000"},
@@ -206,7 +254,8 @@ func TestGenerateConfig(t *testing.T) {
"status_code": 200 "status_code": 200
}] }]
} }
] ],
"logs": {}
}, },
"https": { "https": {
"listen": [":443"], "listen": [":443"],
@@ -215,6 +264,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["secure.example.com"]}], "match": [{"host": ["secure.example.com"]}],
"handle": [{ "handle": [{
"handler": "reverse_proxy", "handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [ "upstreams": [
{"dial": "10.210.0.3:8888"}, {"dial": "10.210.0.3:8888"},
{"dial": "10.210.0.4:8888"}, {"dial": "10.210.0.4:8888"},
@@ -222,7 +279,8 @@ func TestGenerateConfig(t *testing.T) {
] ]
}] }]
} }
] ],
"logs": {}
} }
} }
}`, }`,
@@ -286,6 +344,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["app.example.com"]}], "match": [{"host": ["app.example.com"]}],
"handle": [{ "handle": [{
"handler": "reverse_proxy", "handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [{"dial": "10.210.0.2:8080"}] "upstreams": [{"dial": "10.210.0.2:8080"}]
}] }]
}, },
@@ -297,10 +363,12 @@ func TestGenerateConfig(t *testing.T) {
"status_code": 200 "status_code": 200
}] }]
} }
] ],
"logs": {}
}, },
"https": { "https": {
"listen": [":443"] "listen": [":443"],
"logs": {}
} }
} }
}`, }`,
internal/machine/caddyfile/controller.go → internal/machine/caddyconfig/controller.go
+2 -2
View File
@@ -1,4 +1,4 @@
package caddyfile package caddyconfig
import ( import (
"context" "context"
@@ -19,7 +19,7 @@ const (
) )
// Controller monitors container changes in the cluster store and generates a configuration file for Caddy reverse // Controller monitors container changes in the cluster store and generates a configuration file for Caddy reverse
// proxy. The generated Caddyfile allows Caddy to route external traffic to service containers across the internal // proxy. The generated configuration allows Caddy to route external traffic to service containers across the internal
// network. // network.
type Controller struct { type Controller struct {
store *store.Store store *store.Store
internal/machine/machine.go
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"github.com/psviderski/uncloud/internal/fs" "github.com/psviderski/uncloud/internal/fs"
"github.com/psviderski/uncloud/internal/machine/api/pb" "github.com/psviderski/uncloud/internal/machine/api/pb"
apiproxy "github.com/psviderski/uncloud/internal/machine/api/proxy" apiproxy "github.com/psviderski/uncloud/internal/machine/api/proxy"
"github.com/psviderski/uncloud/internal/machine/caddyfile" "github.com/psviderski/uncloud/internal/machine/caddyconfig"
"github.com/psviderski/uncloud/internal/machine/cluster" "github.com/psviderski/uncloud/internal/machine/cluster"
"github.com/psviderski/uncloud/internal/machine/corroservice" "github.com/psviderski/uncloud/internal/machine/corroservice"
"github.com/psviderski/uncloud/internal/machine/dns" "github.com/psviderski/uncloud/internal/machine/dns"
@@ -384,7 +384,7 @@ func (m *Machine) Run(ctx context.Context) error {
// Create a new Caddyfile controller for managing the Caddy reverse proxy configuration. // Create a new Caddyfile controller for managing the Caddy reverse proxy configuration.
// It will also serve the current machine ID at /.uncloud-verify to verify Caddy reachability. // It will also serve the current machine ID at /.uncloud-verify to verify Caddy reachability.
caddyfileCtrl, err := caddyfile.NewController(m.store, m.config.CaddyConfigPath, m.state.ID) caddyfileCtrl, err := caddyconfig.NewController(m.store, m.config.CaddyConfigPath, m.state.ID)
if err != nil { if err != nil {
return fmt.Errorf("create Caddyfile controller: %w", err) return fmt.Errorf("create Caddyfile controller: %w", err)
} }
internal/machine/network.go
+3 -3
View File
@@ -14,7 +14,7 @@ import (
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
"github.com/docker/docker/client" "github.com/docker/docker/client"
"github.com/psviderski/uncloud/internal/machine/api/pb" "github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/caddyfile" "github.com/psviderski/uncloud/internal/machine/caddyconfig"
"github.com/psviderski/uncloud/internal/machine/corroservice" "github.com/psviderski/uncloud/internal/machine/corroservice"
"github.com/psviderski/uncloud/internal/machine/dns" "github.com/psviderski/uncloud/internal/machine/dns"
"github.com/psviderski/uncloud/internal/machine/docker" "github.com/psviderski/uncloud/internal/machine/docker"
@@ -39,7 +39,7 @@ type networkController struct {
server *grpc.Server server *grpc.Server
corroService corroservice.Service corroService corroservice.Service
dockerCli *client.Client dockerCli *client.Client
caddyfileCtrl *caddyfile.Controller caddyfileCtrl *caddyconfig.Controller
// dnsServer is the embedded internal DNS server for the cluster listening on the machine IP. // dnsServer is the embedded internal DNS server for the cluster listening on the machine IP.
dnsServer *dns.Server dnsServer *dns.Server
@@ -52,7 +52,7 @@ func newNetworkController(
server *grpc.Server, server *grpc.Server,
corroService corroservice.Service, corroService corroservice.Service,
dockerCli *client.Client, dockerCli *client.Client,
caddyfileCtrl *caddyfile.Controller, caddyfileCtrl *caddyconfig.Controller,
dnsServer *dns.Server, dnsServer *dns.Server,
dnsResolver *dns.ClusterResolver, dnsResolver *dns.ClusterResolver,
) ( ) (
pkg/client/dns.go
+2 -2
View File
@@ -7,7 +7,7 @@ import (
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
"github.com/docker/compose/v2/pkg/progress" "github.com/docker/compose/v2/pkg/progress"
"github.com/psviderski/uncloud/internal/machine/api/pb" "github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/caddyfile" "github.com/psviderski/uncloud/internal/machine/caddyconfig"
"github.com/psviderski/uncloud/pkg/api" "github.com/psviderski/uncloud/pkg/api"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
@@ -110,7 +110,7 @@ func verifyCaddyReachable(ctx context.Context, m *pb.MachineInfo) error {
eventID := fmt.Sprintf("Machine %s (%s)", m.Name, publicIP) eventID := fmt.Sprintf("Machine %s (%s)", m.Name, publicIP)
pw.Event(progress.NewEvent(eventID, progress.Working, "Querying")) pw.Event(progress.NewEvent(eventID, progress.Working, "Querying"))
verifyURL := fmt.Sprintf("http://%s%s", publicIP, caddyfile.VerifyPath) verifyURL := fmt.Sprintf("http://%s%s", publicIP, caddyconfig.VerifyPath)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, verifyURL, nil) req, err := http.NewRequestWithContext(ctx, http.MethodGet, verifyURL, nil)
if err != nil { if err != nil {