diff --git a/internal/machine/store/container.go b/internal/machine/store/container.go index c5a5da85..281804ec 100644 --- a/internal/machine/store/container.go +++ b/internal/machine/store/container.go @@ -165,7 +165,7 @@ func (s *Store) DeleteContainers(ctx context.Context, opts DeleteOptions) error // SubscribeContainers returns a list of containers and a channel that signals changes to the list. The channel doesn't // receive any values, it just signals when a container(s) has been added, updated, or deleted in the database. func (s *Store) SubscribeContainers(ctx context.Context) ([]ContainerRecord, <-chan struct{}, error) { - // TODO: figure out whether we need sync_status at all. + // TODO: figure out whether we need sync_status at all (not used at the moment). q := sq.Select("container", "machine_id", "sync_status", "updated_at").From("containers"). Where(sq.Eq{"sync_status": SyncStatusSynced}) query, args, err := q.ToSql() diff --git a/pkg/client/caddy.go b/pkg/client/caddy.go index b4b7de98..2f3e0751 100644 --- a/pkg/client/caddy.go +++ b/pkg/client/caddy.go @@ -35,13 +35,24 @@ func (cli *Client) NewCaddyDeployment(image, config string, placement api.Placem spec := api.ServiceSpec{ Container: api.ContainerSpec{ - Command: []string{"caddy", "run", "-c", "/config/caddy.json", "--watch"}, - Image: image, + Command: []string{"caddy", "run", "-c", "/config/Caddyfile", "--watch"}, + Env: map[string]string{ + "CADDY_ADMIN": "unix//run/caddy/admin.sock", + }, + Image: image, VolumeMounts: []api.VolumeMount{ { - VolumeName: "config", + VolumeName: "data", ContainerPath: "/config", }, + { + VolumeName: "data", + ContainerPath: "/data", + }, + { + VolumeName: "run", + ContainerPath: "/run/caddy", + }, }, }, Mode: api.ServiceModeGlobal, @@ -63,12 +74,20 @@ func (cli *Client) NewCaddyDeployment(image, config string, placement api.Placem }, Volumes: []api.VolumeSpec{ { - Name: "config", + Name: "data", Type: api.VolumeTypeBind, BindOptions: &api.BindOptions{ HostPath: "/var/lib/uncloud/caddy", }, }, + { + Name: "run", + Type: api.VolumeTypeBind, + BindOptions: &api.BindOptions{ + HostPath: "/run/uncloud/caddy", + CreateHostPath: true, + }, + }, }, } diff --git a/test/e2e/service_test.go b/test/e2e/service_test.go index 7d7ccfae..d97f8bbd 100644 --- a/test/e2e/service_test.go +++ b/test/e2e/service_test.go @@ -4,12 +4,15 @@ import ( "context" "errors" "net/netip" + "strings" "testing" + "time" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/volume" "github.com/docker/go-units" + "github.com/psviderski/uncloud/internal/machine/api/pb" "github.com/psviderski/uncloud/internal/secret" "github.com/psviderski/uncloud/internal/ucind" "github.com/psviderski/uncloud/pkg/api" @@ -338,38 +341,106 @@ func TestDeployment(t *testing.T) { // assert.True(t, containers.Contains(initialContainerID), "Expected initial container to remain") }) - t.Run("caddy with custom config", func(t *testing.T) { + t.Run("caddy and service with custom configs", func(t *testing.T) { + name := "test-custom-caddy-config" t.Cleanup(func() { - err := cli.RemoveService(ctx, client.CaddyServiceName) + err := cli.RemoveService(ctx, name) + if !errors.Is(err, api.ErrNotFound) { + require.NoError(t, err) + } + err = cli.RemoveService(ctx, client.CaddyServiceName) if !errors.Is(err, api.ErrNotFound) { require.NoError(t, err) } }) - caddyfile := `{ + // First deploy a service with custom caddy config before caddy is deployed. + serviceCaddyfile := `test-custom-caddy-config.example.com { + reverse_proxy {{upstreams}} { + import common_proxy + } + log +}` + spec := api.ServiceSpec{ + Name: name, + Container: api.ContainerSpec{ + Image: "portainer/pause:latest", + }, + Caddy: &api.CaddySpec{ + Config: serviceCaddyfile, + }, + } + + deployment := cli.NewDeployment(spec, nil) + _, err := deployment.Run(ctx) + require.NoError(t, err) + + svc, err := cli.InspectService(ctx, name) + require.NoError(t, err) + assertServiceMatchesSpec(t, svc, spec) + + // Check that the generated Caddyfile contains a comment with invalid user-defined configs. + var config *pb.GetCaddyConfigResponse + require.Eventually(t, func() bool { + config, err = cli.Caddy.GetConfig(ctx, nil) + if err != nil { + return false + } + return strings.Contains(config.Caddyfile, "invalid user-defined configs") + }, 5*time.Second, 100*time.Millisecond) + + assert.Regexp(t, "- service 'test-custom-caddy-config': validation failed:.*"+ + "/run/uncloud/caddy/admin.sock: connect:.*", config.Caddyfile, + "Expected comment about validation failure for service's user-defined Caddy config") + assert.NotContains(t, config.Caddyfile, "test-custom-caddy-config.example.com {") + + // Now deploy caddy with custom config. + caddyCaddyfile := `{ debug } myapp.example.com { - reverse_proxy myapp:8000 + reverse_proxy 1.2.3.4:8000 }` - deployment, err := cli.NewCaddyDeployment("", caddyfile, api.Placement{}) + caddyDeployment, err := cli.NewCaddyDeployment("", caddyCaddyfile, api.Placement{}) require.NoError(t, err) - _, err = deployment.Run(ctx) + _, err = caddyDeployment.Run(ctx) require.NoError(t, err) - svc, err := cli.InspectService(ctx, client.CaddyServiceName) + caddySvc, err := cli.InspectService(ctx, client.CaddyServiceName) require.NoError(t, err) - assertServiceMatchesSpec(t, svc, deployment.Spec) + assertServiceMatchesSpec(t, caddySvc, caddyDeployment.Spec) - config, err := cli.Caddy.GetConfig(ctx, nil) - require.NoError(t, err) + // Wait for the Caddyfile to be regenerated with both custom configs. + require.Eventually(t, func() bool { + config, err = cli.Caddy.GetConfig(ctx, nil) + if err != nil { + return false + } + // Both configs should be present. + return strings.Contains(config.Caddyfile, caddyCaddyfile) && + strings.Contains(config.Caddyfile, "test-custom-caddy-config.example.com") + }, 5*time.Second, 100*time.Millisecond, + "Expected both custom configs to be included in the Caddyfile") assert.Contains(t, config.Caddyfile, "# This file is autogenerated by Uncloud") assert.Contains(t, config.Caddyfile, "handle /.uncloud-verify") - assert.Contains(t, config.Caddyfile, caddyfile, + assert.Contains(t, config.Caddyfile, caddyCaddyfile, "Expected user-defined global Caddy config to be included in the Caddyfile") + + ctrIP := svc.Containers[0].Container.UncloudNetworkIP().String() + renderedServiceCaddyfile := `test-custom-caddy-config.example.com { + reverse_proxy ` + ctrIP + ` { + import common_proxy + } + log +}` + assert.Contains(t, config.Caddyfile, renderedServiceCaddyfile, + "Expected rendered user-defined Caddy config for test service to be included in the Caddyfile") + + assert.NotContains(t, config.Caddyfile, "invalid user-defined configs", + "Should not have validation failure comments after caddy is deployed") }) t.Run("replicated", func(t *testing.T) {