feat: migrate Caddy to generated Caddyfile, mount persistent data volume

internal/machine/store/container.go

@@ -165,7 +165,7 @@ func (s *Store) DeleteContainers(ctx context.Context, opts DeleteOptions) error
 // SubscribeContainers returns a list of containers and a channel that signals changes to the list. The channel doesn't
 // receive any values, it just signals when a container(s) has been added, updated, or deleted in the database.
 func (s *Store) SubscribeContainers(ctx context.Context) ([]ContainerRecord, <-chan struct{}, error) {
-	// TODO: figure out whether we need sync_status at all.
+	// TODO: figure out whether we need sync_status at all (not used at the moment).
 	q := sq.Select("container", "machine_id", "sync_status", "updated_at").From("containers").
 		Where(sq.Eq{"sync_status": SyncStatusSynced})
 	query, args, err := q.ToSql()

pkg/client/caddy.go

@@ -35,13 +35,24 @@ func (cli *Client) NewCaddyDeployment(image, config string, placement api.Placem
 
 	spec := api.ServiceSpec{
 		Container: api.ContainerSpec{
-			Command: []string{"caddy", "run", "-c", "/config/caddy.json", "--watch"},
+			Command: []string{"caddy", "run", "-c", "/config/Caddyfile", "--watch"},
+			Env: map[string]string{
+				"CADDY_ADMIN": "unix//run/caddy/admin.sock",
+			},
 			Image: image,
 			VolumeMounts: []api.VolumeMount{
 				{
-					VolumeName:    "config",
+					VolumeName:    "data",
 					ContainerPath: "/config",
 				},
+				{
+					VolumeName:    "data",
+					ContainerPath: "/data",
+				},
+				{
+					VolumeName:    "run",
+					ContainerPath: "/run/caddy",
+				},
 			},
 		},
 		Mode:      api.ServiceModeGlobal,
@@ -63,12 +74,20 @@ func (cli *Client) NewCaddyDeployment(image, config string, placement api.Placem
 		},
 		Volumes: []api.VolumeSpec{
 			{
-				Name: "config",
+				Name: "data",
 				Type: api.VolumeTypeBind,
 				BindOptions: &api.BindOptions{
 					HostPath: "/var/lib/uncloud/caddy",
 				},
 			},
+			{
+				Name: "run",
+				Type: api.VolumeTypeBind,
+				BindOptions: &api.BindOptions{
+					HostPath:       "/run/uncloud/caddy",
+					CreateHostPath: true,
+				},
+			},
 		},
 	}
 

test/e2e/service_test.go

@@ -4,12 +4,15 @@ import (
 	"context"
 	"errors"
 	"net/netip"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/go-units"
+	"github.com/psviderski/uncloud/internal/machine/api/pb"
 	"github.com/psviderski/uncloud/internal/secret"
 	"github.com/psviderski/uncloud/internal/ucind"
 	"github.com/psviderski/uncloud/pkg/api"
@@ -338,38 +341,106 @@ func TestDeployment(t *testing.T) {
 		// assert.True(t, containers.Contains(initialContainerID), "Expected initial container to remain")
 	})
 
-	t.Run("caddy with custom config", func(t *testing.T) {
+	t.Run("caddy and service with custom configs", func(t *testing.T) {
+		name := "test-custom-caddy-config"
 		t.Cleanup(func() {
-			err := cli.RemoveService(ctx, client.CaddyServiceName)
+			err := cli.RemoveService(ctx, name)
+			if !errors.Is(err, api.ErrNotFound) {
+				require.NoError(t, err)
+			}
+			err = cli.RemoveService(ctx, client.CaddyServiceName)
 			if !errors.Is(err, api.ErrNotFound) {
 				require.NoError(t, err)
 			}
 		})
 
-		caddyfile := `{
+		// First deploy a service with custom caddy config before caddy is deployed.
+		serviceCaddyfile := `test-custom-caddy-config.example.com {
+	reverse_proxy {{upstreams}} {
+		import common_proxy
+	}
+	log
+}`
+		spec := api.ServiceSpec{
+			Name: name,
+			Container: api.ContainerSpec{
+				Image: "portainer/pause:latest",
+			},
+			Caddy: &api.CaddySpec{
+				Config: serviceCaddyfile,
+			},
+		}
+
+		deployment := cli.NewDeployment(spec, nil)
+		_, err := deployment.Run(ctx)
+		require.NoError(t, err)
+
+		svc, err := cli.InspectService(ctx, name)
+		require.NoError(t, err)
+		assertServiceMatchesSpec(t, svc, spec)
+
+		// Check that the generated Caddyfile contains a comment with invalid user-defined configs.
+		var config *pb.GetCaddyConfigResponse
+		require.Eventually(t, func() bool {
+			config, err = cli.Caddy.GetConfig(ctx, nil)
+			if err != nil {
+				return false
+			}
+			return strings.Contains(config.Caddyfile, "invalid user-defined configs")
+		}, 5*time.Second, 100*time.Millisecond)
+
+		assert.Regexp(t, "- service 'test-custom-caddy-config': validation failed:.*"+
+			"/run/uncloud/caddy/admin.sock: connect:.*", config.Caddyfile,
+			"Expected comment about validation failure for service's user-defined Caddy config")
+		assert.NotContains(t, config.Caddyfile, "test-custom-caddy-config.example.com {")
+
+		// Now deploy caddy with custom config.
+		caddyCaddyfile := `{
 	debug
 }
 
 myapp.example.com {
-	reverse_proxy myapp:8000
+	reverse_proxy 1.2.3.4:8000
 }`
-		deployment, err := cli.NewCaddyDeployment("", caddyfile, api.Placement{})
+		caddyDeployment, err := cli.NewCaddyDeployment("", caddyCaddyfile, api.Placement{})
 		require.NoError(t, err)
 
-		_, err = deployment.Run(ctx)
+		_, err = caddyDeployment.Run(ctx)
 		require.NoError(t, err)
 
-		svc, err := cli.InspectService(ctx, client.CaddyServiceName)
+		caddySvc, err := cli.InspectService(ctx, client.CaddyServiceName)
 		require.NoError(t, err)
-		assertServiceMatchesSpec(t, svc, deployment.Spec)
+		assertServiceMatchesSpec(t, caddySvc, caddyDeployment.Spec)
 
-		config, err := cli.Caddy.GetConfig(ctx, nil)
-		require.NoError(t, err)
+		// Wait for the Caddyfile to be regenerated with both custom configs.
+		require.Eventually(t, func() bool {
+			config, err = cli.Caddy.GetConfig(ctx, nil)
+			if err != nil {
+				return false
+			}
+			// Both configs should be present.
+			return strings.Contains(config.Caddyfile, caddyCaddyfile) &&
+				strings.Contains(config.Caddyfile, "test-custom-caddy-config.example.com")
+		}, 5*time.Second, 100*time.Millisecond,
+			"Expected both custom configs to be included in the Caddyfile")
 
 		assert.Contains(t, config.Caddyfile, "# This file is autogenerated by Uncloud")
 		assert.Contains(t, config.Caddyfile, "handle /.uncloud-verify")
-		assert.Contains(t, config.Caddyfile, caddyfile,
+		assert.Contains(t, config.Caddyfile, caddyCaddyfile,
 			"Expected user-defined global Caddy config to be included in the Caddyfile")
+
+		ctrIP := svc.Containers[0].Container.UncloudNetworkIP().String()
+		renderedServiceCaddyfile := `test-custom-caddy-config.example.com {
+	reverse_proxy ` + ctrIP + ` {
+		import common_proxy
+	}
+	log
+}`
+		assert.Contains(t, config.Caddyfile, renderedServiceCaddyfile,
+			"Expected rendered user-defined Caddy config for test service to be included in the Caddyfile")
+
+		assert.NotContains(t, config.Caddyfile, "invalid user-defined configs",
+			"Should not have validation failure comments after caddy is deployed")
 	})
 
 	t.Run("replicated", func(t *testing.T) {