philipp/uncloud
1
0
Fork 0
mirror of https://github.com/psviderski/uncloud.git synced 2026-08-26 11:03:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: enable passive health checks and upstream retries in Caddy

Browse Source
This commit is contained in:
Pavel Sviderski
2025-06-12 18:38:28 +10:00
parent 2a0d31cec0
commit fc61e2fc07
6 changed files with 106 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/machine/caddyfile/config.go → internal/machine/caddyconfig/config.go
+16 -1
View File
@@ -1,4 +1,4 @@
package caddyfile
package caddyconfig
import (
"encoding/json"
@@ -10,6 +10,7 @@ import (
"net/http"
"slices"
"strconv"
"time"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
@@ -64,6 +65,8 @@ func GenerateConfig(containers []api.ServiceContainer, verifyResponse string) (*
servers := make(map[string]*caddyhttp.Server)
servers["http"] = &caddyhttp.Server{
Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)},
// All http requests to this server are logged to the default logger.
Logs: &caddyhttp.ServerLogConfig{},
Routes: append(
hostUpstreamsToRoutes(httpHostUpstreams, &warnings),
// Add a route to respond with a static verification response at the /.uncloud-verify path.
@@ -72,6 +75,8 @@ func GenerateConfig(containers []api.ServiceContainer, verifyResponse string) (*
}
servers["https"] = &caddyhttp.Server{
Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)},
// All https requests to this server are logged to the default logger.
Logs: &caddyhttp.ServerLogConfig{},
Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings),
}
@@ -112,6 +117,16 @@ func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyc
}
}
handler := &reverseproxy.Handler{
HealthChecks: &reverseproxy.HealthChecks{
// Enable passive health checks to automatically detect unhealthy upstreams.
Passive: &reverseproxy.PassiveHealthChecks{
FailDuration: caddy.Duration(30 * time.Second),
},
},
LoadBalancing: &reverseproxy.LoadBalancing{
// Retry failed requests to skip over temporarily unavailable upstreams.
Retries: 3,
},
Upstreams: upstreamPool,
}
internal/machine/caddyfile/config_test.go → internal/machine/caddyconfig/config_test.go
+81 -13
View File
@@ -1,4 +1,4 @@
package caddyfile
package caddyconfig
import (
"strings"
@@ -25,10 +25,12 @@ func TestGenerateConfig(t *testing.T) {
"handler": "static_response",
"status_code": 200
}]
}]
}],
"logs": {}
},
"https": {
"listen": [":443"]
"listen": [":443"],
"logs": {}
}
}
}`
@@ -60,6 +62,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["app.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [{"dial": "10.210.0.2:8080"}]
}]
},
@@ -71,10 +81,12 @@ func TestGenerateConfig(t *testing.T) {
"status_code": 200
}]
}
]
],
"logs": {}
},
"https": {
"listen": [":443"]
"listen": [":443"],
"logs": {}
}
}
}`,
@@ -95,6 +107,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["app.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [
{"dial": "10.210.0.2:8080"},
{"dial": "10.210.0.3:8080"}
@@ -109,10 +129,12 @@ func TestGenerateConfig(t *testing.T) {
"status_code": 200
}]
}
]
],
"logs": {}
},
"https": {
"listen": [":443"]
"listen": [":443"],
"logs": {}
}
}
}`,
@@ -136,7 +158,8 @@ func TestGenerateConfig(t *testing.T) {
"status_code": 200
}]
}
]
],
"logs": {}
},
"https": {
"listen": [":443"],
@@ -145,10 +168,19 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["secure.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [{"dial": "10.210.0.2:8000"}]
}]
}
]
],
"logs": {}
}
}
}`,
@@ -180,6 +212,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["app.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [
{"dial": "10.210.0.2:8080"},
{"dial": "10.210.0.3:8080"},
@@ -191,6 +231,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["web.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [
{"dial": "10.210.0.2:8000"},
{"dial": "10.210.0.4:8000"},
@@ -206,7 +254,8 @@ func TestGenerateConfig(t *testing.T) {
"status_code": 200
}]
}
]
],
"logs": {}
},
"https": {
"listen": [":443"],
@@ -215,6 +264,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["secure.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [
{"dial": "10.210.0.3:8888"},
{"dial": "10.210.0.4:8888"},
@@ -222,7 +279,8 @@ func TestGenerateConfig(t *testing.T) {
]
}]
}
]
],
"logs": {}
}
}
}`,
@@ -286,6 +344,14 @@ func TestGenerateConfig(t *testing.T) {
"match": [{"host": ["app.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [{"dial": "10.210.0.2:8080"}]
}]
},
@@ -297,10 +363,12 @@ func TestGenerateConfig(t *testing.T) {
"status_code": 200
}]
}
]
],
"logs": {}
},
"https": {
"listen": [":443"]
"listen": [":443"],
"logs": {}
}
}
}`,
internal/machine/caddyfile/controller.go → internal/machine/caddyconfig/controller.go
+2 -2
View File
@@ -1,4 +1,4 @@
package caddyfile
package caddyconfig
import (
"context"
@@ -19,7 +19,7 @@ const (
)
// Controller monitors container changes in the cluster store and generates a configuration file for Caddy reverse
// proxy. The generated Caddyfile allows Caddy to route external traffic to service containers across the internal
// proxy. The generated configuration allows Caddy to route external traffic to service containers across the internal
// network.
type Controller struct {
store *store.Store
internal/machine/machine.go
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"github.com/psviderski/uncloud/internal/fs"
"github.com/psviderski/uncloud/internal/machine/api/pb"
apiproxy "github.com/psviderski/uncloud/internal/machine/api/proxy"
"github.com/psviderski/uncloud/internal/machine/caddyfile"
"github.com/psviderski/uncloud/internal/machine/caddyconfig"
"github.com/psviderski/uncloud/internal/machine/cluster"
"github.com/psviderski/uncloud/internal/machine/corroservice"
"github.com/psviderski/uncloud/internal/machine/dns"
@@ -384,7 +384,7 @@ func (m *Machine) Run(ctx context.Context) error {
// Create a new Caddyfile controller for managing the Caddy reverse proxy configuration.
// It will also serve the current machine ID at /.uncloud-verify to verify Caddy reachability.
caddyfileCtrl, err := caddyfile.NewController(m.store, m.config.CaddyConfigPath, m.state.ID)
caddyfileCtrl, err := caddyconfig.NewController(m.store, m.config.CaddyConfigPath, m.state.ID)
if err != nil {
return fmt.Errorf("create Caddyfile controller: %w", err)
}
internal/machine/network.go
+3 -3
View File
@@ -14,7 +14,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/docker/docker/client"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/caddyfile"
"github.com/psviderski/uncloud/internal/machine/caddyconfig"
"github.com/psviderski/uncloud/internal/machine/corroservice"
"github.com/psviderski/uncloud/internal/machine/dns"
"github.com/psviderski/uncloud/internal/machine/docker"
@@ -39,7 +39,7 @@ type networkController struct {
server *grpc.Server
corroService corroservice.Service
dockerCli *client.Client
caddyfileCtrl *caddyfile.Controller
caddyfileCtrl *caddyconfig.Controller
// dnsServer is the embedded internal DNS server for the cluster listening on the machine IP.
dnsServer *dns.Server
@@ -52,7 +52,7 @@ func newNetworkController(
server *grpc.Server,
corroService corroservice.Service,
dockerCli *client.Client,
caddyfileCtrl *caddyfile.Controller,
caddyfileCtrl *caddyconfig.Controller,
dnsServer *dns.Server,
dnsResolver *dns.ClusterResolver,
) (
pkg/client/dns.go
+2 -2
View File
@@ -7,7 +7,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/docker/compose/v2/pkg/progress"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/caddyfile"
"github.com/psviderski/uncloud/internal/machine/caddyconfig"
"github.com/psviderski/uncloud/pkg/api"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -110,7 +110,7 @@ func verifyCaddyReachable(ctx context.Context, m *pb.MachineInfo) error {
eventID := fmt.Sprintf("Machine %s (%s)", m.Name, publicIP)
pw.Event(progress.NewEvent(eventID, progress.Working, "Querying"))
verifyURL := fmt.Sprintf("http://%s%s", publicIP, caddyfile.VerifyPath)
verifyURL := fmt.Sprintf("http://%s%s", publicIP, caddyconfig.VerifyPath)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, verifyURL, nil)
if err != nil {