diff --git a/internal/machine/caddyfile/config.go b/internal/machine/caddyconfig/config.go similarity index 88% rename from internal/machine/caddyfile/config.go rename to internal/machine/caddyconfig/config.go index 31fcc0ba..c6f6485a 100644 --- a/internal/machine/caddyfile/config.go +++ b/internal/machine/caddyconfig/config.go @@ -1,4 +1,4 @@ -package caddyfile +package caddyconfig import ( "encoding/json" @@ -10,6 +10,7 @@ import ( "net/http" "slices" "strconv" + "time" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig" @@ -64,6 +65,8 @@ func GenerateConfig(containers []api.ServiceContainer, verifyResponse string) (* servers := make(map[string]*caddyhttp.Server) servers["http"] = &caddyhttp.Server{ Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)}, + // All http requests to this server are logged to the default logger. + Logs: &caddyhttp.ServerLogConfig{}, Routes: append( hostUpstreamsToRoutes(httpHostUpstreams, &warnings), // Add a route to respond with a static verification response at the /.uncloud-verify path. @@ -72,6 +75,8 @@ func GenerateConfig(containers []api.ServiceContainer, verifyResponse string) (* } servers["https"] = &caddyhttp.Server{ Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)}, + // All https requests to this server are logged to the default logger. + Logs: &caddyhttp.ServerLogConfig{}, Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings), } @@ -112,6 +117,16 @@ func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyc } } handler := &reverseproxy.Handler{ + HealthChecks: &reverseproxy.HealthChecks{ + // Enable passive health checks to automatically detect unhealthy upstreams. + Passive: &reverseproxy.PassiveHealthChecks{ + FailDuration: caddy.Duration(30 * time.Second), + }, + }, + LoadBalancing: &reverseproxy.LoadBalancing{ + // Retry failed requests to skip over temporarily unavailable upstreams. + Retries: 3, + }, Upstreams: upstreamPool, } diff --git a/internal/machine/caddyfile/config_test.go b/internal/machine/caddyconfig/config_test.go similarity index 78% rename from internal/machine/caddyfile/config_test.go rename to internal/machine/caddyconfig/config_test.go index fa865ea7..ebc139e4 100644 --- a/internal/machine/caddyfile/config_test.go +++ b/internal/machine/caddyconfig/config_test.go @@ -1,4 +1,4 @@ -package caddyfile +package caddyconfig import ( "strings" @@ -25,10 +25,12 @@ func TestGenerateConfig(t *testing.T) { "handler": "static_response", "status_code": 200 }] - }] + }], + "logs": {} }, "https": { - "listen": [":443"] + "listen": [":443"], + "logs": {} } } }` @@ -60,6 +62,14 @@ func TestGenerateConfig(t *testing.T) { "match": [{"host": ["app.example.com"]}], "handle": [{ "handler": "reverse_proxy", + "health_checks": { + "passive": { + "fail_duration": 30000000000 + } + }, + "load_balancing": { + "retries": 3 + }, "upstreams": [{"dial": "10.210.0.2:8080"}] }] }, @@ -71,10 +81,12 @@ func TestGenerateConfig(t *testing.T) { "status_code": 200 }] } - ] + ], + "logs": {} }, "https": { - "listen": [":443"] + "listen": [":443"], + "logs": {} } } }`, @@ -95,6 +107,14 @@ func TestGenerateConfig(t *testing.T) { "match": [{"host": ["app.example.com"]}], "handle": [{ "handler": "reverse_proxy", + "health_checks": { + "passive": { + "fail_duration": 30000000000 + } + }, + "load_balancing": { + "retries": 3 + }, "upstreams": [ {"dial": "10.210.0.2:8080"}, {"dial": "10.210.0.3:8080"} @@ -109,10 +129,12 @@ func TestGenerateConfig(t *testing.T) { "status_code": 200 }] } - ] + ], + "logs": {} }, "https": { - "listen": [":443"] + "listen": [":443"], + "logs": {} } } }`, @@ -136,7 +158,8 @@ func TestGenerateConfig(t *testing.T) { "status_code": 200 }] } - ] + ], + "logs": {} }, "https": { "listen": [":443"], @@ -145,10 +168,19 @@ func TestGenerateConfig(t *testing.T) { "match": [{"host": ["secure.example.com"]}], "handle": [{ "handler": "reverse_proxy", + "health_checks": { + "passive": { + "fail_duration": 30000000000 + } + }, + "load_balancing": { + "retries": 3 + }, "upstreams": [{"dial": "10.210.0.2:8000"}] }] } - ] + ], + "logs": {} } } }`, @@ -180,6 +212,14 @@ func TestGenerateConfig(t *testing.T) { "match": [{"host": ["app.example.com"]}], "handle": [{ "handler": "reverse_proxy", + "health_checks": { + "passive": { + "fail_duration": 30000000000 + } + }, + "load_balancing": { + "retries": 3 + }, "upstreams": [ {"dial": "10.210.0.2:8080"}, {"dial": "10.210.0.3:8080"}, @@ -191,6 +231,14 @@ func TestGenerateConfig(t *testing.T) { "match": [{"host": ["web.example.com"]}], "handle": [{ "handler": "reverse_proxy", + "health_checks": { + "passive": { + "fail_duration": 30000000000 + } + }, + "load_balancing": { + "retries": 3 + }, "upstreams": [ {"dial": "10.210.0.2:8000"}, {"dial": "10.210.0.4:8000"}, @@ -206,7 +254,8 @@ func TestGenerateConfig(t *testing.T) { "status_code": 200 }] } - ] + ], + "logs": {} }, "https": { "listen": [":443"], @@ -215,6 +264,14 @@ func TestGenerateConfig(t *testing.T) { "match": [{"host": ["secure.example.com"]}], "handle": [{ "handler": "reverse_proxy", + "health_checks": { + "passive": { + "fail_duration": 30000000000 + } + }, + "load_balancing": { + "retries": 3 + }, "upstreams": [ {"dial": "10.210.0.3:8888"}, {"dial": "10.210.0.4:8888"}, @@ -222,7 +279,8 @@ func TestGenerateConfig(t *testing.T) { ] }] } - ] + ], + "logs": {} } } }`, @@ -286,6 +344,14 @@ func TestGenerateConfig(t *testing.T) { "match": [{"host": ["app.example.com"]}], "handle": [{ "handler": "reverse_proxy", + "health_checks": { + "passive": { + "fail_duration": 30000000000 + } + }, + "load_balancing": { + "retries": 3 + }, "upstreams": [{"dial": "10.210.0.2:8080"}] }] }, @@ -297,10 +363,12 @@ func TestGenerateConfig(t *testing.T) { "status_code": 200 }] } - ] + ], + "logs": {} }, "https": { - "listen": [":443"] + "listen": [":443"], + "logs": {} } } }`, diff --git a/internal/machine/caddyfile/controller.go b/internal/machine/caddyconfig/controller.go similarity index 96% rename from internal/machine/caddyfile/controller.go rename to internal/machine/caddyconfig/controller.go index e82f8c9d..13b70008 100644 --- a/internal/machine/caddyfile/controller.go +++ b/internal/machine/caddyconfig/controller.go @@ -1,4 +1,4 @@ -package caddyfile +package caddyconfig import ( "context" @@ -19,7 +19,7 @@ const ( ) // Controller monitors container changes in the cluster store and generates a configuration file for Caddy reverse -// proxy. The generated Caddyfile allows Caddy to route external traffic to service containers across the internal +// proxy. The generated configuration allows Caddy to route external traffic to service containers across the internal // network. type Controller struct { store *store.Store diff --git a/internal/machine/machine.go b/internal/machine/machine.go index e6e07b7e..07fd5bca 100644 --- a/internal/machine/machine.go +++ b/internal/machine/machine.go @@ -21,7 +21,7 @@ import ( "github.com/psviderski/uncloud/internal/fs" "github.com/psviderski/uncloud/internal/machine/api/pb" apiproxy "github.com/psviderski/uncloud/internal/machine/api/proxy" - "github.com/psviderski/uncloud/internal/machine/caddyfile" + "github.com/psviderski/uncloud/internal/machine/caddyconfig" "github.com/psviderski/uncloud/internal/machine/cluster" "github.com/psviderski/uncloud/internal/machine/corroservice" "github.com/psviderski/uncloud/internal/machine/dns" @@ -384,7 +384,7 @@ func (m *Machine) Run(ctx context.Context) error { // Create a new Caddyfile controller for managing the Caddy reverse proxy configuration. // It will also serve the current machine ID at /.uncloud-verify to verify Caddy reachability. - caddyfileCtrl, err := caddyfile.NewController(m.store, m.config.CaddyConfigPath, m.state.ID) + caddyfileCtrl, err := caddyconfig.NewController(m.store, m.config.CaddyConfigPath, m.state.ID) if err != nil { return fmt.Errorf("create Caddyfile controller: %w", err) } diff --git a/internal/machine/network.go b/internal/machine/network.go index 5046ba35..8e8de87a 100644 --- a/internal/machine/network.go +++ b/internal/machine/network.go @@ -14,7 +14,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/docker/docker/client" "github.com/psviderski/uncloud/internal/machine/api/pb" - "github.com/psviderski/uncloud/internal/machine/caddyfile" + "github.com/psviderski/uncloud/internal/machine/caddyconfig" "github.com/psviderski/uncloud/internal/machine/corroservice" "github.com/psviderski/uncloud/internal/machine/dns" "github.com/psviderski/uncloud/internal/machine/docker" @@ -39,7 +39,7 @@ type networkController struct { server *grpc.Server corroService corroservice.Service dockerCli *client.Client - caddyfileCtrl *caddyfile.Controller + caddyfileCtrl *caddyconfig.Controller // dnsServer is the embedded internal DNS server for the cluster listening on the machine IP. dnsServer *dns.Server @@ -52,7 +52,7 @@ func newNetworkController( server *grpc.Server, corroService corroservice.Service, dockerCli *client.Client, - caddyfileCtrl *caddyfile.Controller, + caddyfileCtrl *caddyconfig.Controller, dnsServer *dns.Server, dnsResolver *dns.ClusterResolver, ) ( diff --git a/pkg/client/dns.go b/pkg/client/dns.go index 5e986a4d..95130e18 100644 --- a/pkg/client/dns.go +++ b/pkg/client/dns.go @@ -7,7 +7,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/docker/compose/v2/pkg/progress" "github.com/psviderski/uncloud/internal/machine/api/pb" - "github.com/psviderski/uncloud/internal/machine/caddyfile" + "github.com/psviderski/uncloud/internal/machine/caddyconfig" "github.com/psviderski/uncloud/pkg/api" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -110,7 +110,7 @@ func verifyCaddyReachable(ctx context.Context, m *pb.MachineInfo) error { eventID := fmt.Sprintf("Machine %s (%s)", m.Name, publicIP) pw.Event(progress.NewEvent(eventID, progress.Working, "Querying")) - verifyURL := fmt.Sprintf("http://%s%s", publicIP, caddyfile.VerifyPath) + verifyURL := fmt.Sprintf("http://%s%s", publicIP, caddyconfig.VerifyPath) req, err := http.NewRequestWithContext(ctx, http.MethodGet, verifyURL, nil) if err != nil {