feat: deploy caddy service when adding new machine to cluster

cmd/uncloud/machine/add.go

@@ -1,14 +1,25 @@
 package machine
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"github.com/cenkalti/backoff/v4"
+	"github.com/docker/compose/v2/pkg/progress"
 	"github.com/spf13/cobra"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/emptypb"
+	"time"
 	"uncloud/internal/cli"
+	"uncloud/internal/cli/client"
 	"uncloud/internal/cli/config"
+	"uncloud/internal/machine/api/pb"
 )
 
 type addOptions struct {
 	name    string
+	noCaddy bool
 	sshKey  string
 	cluster string
 }
@@ -33,10 +44,14 @@ func NewAddCommand() *cobra.Command {
 				KeyPath: opts.sshKey,
 			}
 
-			return uncli.AddMachine(cmd.Context(), remoteMachine, opts.cluster, opts.name)
+			return add(cmd.Context(), uncli, remoteMachine, opts)
 		},
 	}
 	cmd.Flags().StringVarP(&opts.name, "name", "n", "", "Assign a name to the machine.")
+	cmd.Flags().BoolVar(
+		&opts.noCaddy, "no-caddy", false,
+		"Don't deploy Caddy reverse proxy service to the machine.",
+	)
 	cmd.Flags().StringVarP(
 		&opts.sshKey, "ssh-key", "i", "",
 		"path to SSH private key for SSH remote login. (default ~/.ssh/id_*)",
@@ -47,3 +62,87 @@ func NewAddCommand() *cobra.Command {
 	)
 	return cmd
 }
+
+func add(ctx context.Context, uncli *cli.CLI, remoteMachine cli.RemoteMachine, opts addOptions) error {
+	machineClient, err := uncli.AddMachine(ctx, remoteMachine, opts.cluster, opts.name)
+	if err != nil {
+		return err
+	}
+	defer machineClient.Close()
+
+	if opts.noCaddy {
+		return nil
+	}
+
+	// Wait for the cluster to be initialised to be able to deploy the Caddy service.
+	fmt.Println("Waiting for the machine to be ready...")
+	if err = waitClusterInitialised(ctx, machineClient); err != nil {
+		return fmt.Errorf("wait for cluster to be initialised on machine: %w", err)
+	}
+
+	// Inspect the added machine to get its ID to create a filter for the Caddy deployment.
+	minfo, err := machineClient.Inspect(ctx, &emptypb.Empty{})
+	if err != nil {
+		return fmt.Errorf("inspect machine: %w", err)
+	}
+	filter := func(m *pb.MachineInfo) bool {
+		return m.Id == minfo.Id
+	}
+	// Deploy a Caddy service container to the added machine. If caddy service is already deployed on other machines,
+	// use the deployed image version. Otherwise, use the latest version.
+	caddyImage := ""
+	caddySvc, err := machineClient.InspectService(ctx, client.CaddyServiceName)
+	if err != nil {
+		if !errors.Is(err, client.ErrNotFound) {
+			return fmt.Errorf("inspect caddy service: %w", err)
+		}
+	} else {
+		caddyImage = caddySvc.Containers[0].Container.Config.Image
+		// Find the latest created container and use its image.
+		var latestCreated time.Time
+		for _, c := range caddySvc.Containers[1:] {
+			created, err := time.Parse(time.RFC3339Nano, c.Container.Created)
+			if err != nil {
+				continue
+			}
+			if created.After(latestCreated) {
+				latestCreated = created
+				caddyImage = c.Container.Config.Image
+			}
+		}
+	}
+
+	d, err := machineClient.NewCaddyDeployment(caddyImage, filter)
+	if err != nil {
+		return fmt.Errorf("create caddy deployment: %w", err)
+	}
+
+	return progress.RunWithTitle(ctx, func(ctx context.Context) error {
+		if _, err = d.Run(ctx); err != nil {
+			return fmt.Errorf("deploy caddy: %w", err)
+		}
+		return nil
+	}, uncli.ProgressOut(), fmt.Sprintf("Deploying service %s", d.Spec.Name))
+}
+
+func waitClusterInitialised(ctx context.Context, client *client.Client) error {
+	boff := backoff.WithContext(backoff.NewExponentialBackOff(
+		backoff.WithMaxInterval(1*time.Second),
+		backoff.WithMaxElapsedTime(30*time.Second),
+	), ctx)
+
+	check := func() error {
+		_, err := client.ListMachines(ctx)
+		if err == nil {
+			return nil
+		}
+
+		statusErr := status.Convert(err)
+		if statusErr.Code() == codes.FailedPrecondition {
+			return err
+		}
+		return backoff.Permanent(err)
+	}
+
+	return backoff.Retry(check, boff)
+}

cmd/uncloud/machine/init.go

@@ -89,7 +89,7 @@ func initCluster(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteM
 	// Deploy the Caddy service to the initialised machine.
 	// The creation of a deployment plan talks to cluster API. Since the API needs a few moments to become available
 	// after cluster initialisation, we keep the user informed during this wait.
-	fmt.Println("Waiting for the cluster to be ready...")
+	fmt.Println("Waiting for the machine to be ready...")
 
 	d, err := client.NewCaddyDeployment("", nil)
 	if err != nil {

internal/cli/cli.go

@@ -185,39 +185,44 @@ func (cli *CLI) initRemoteMachine(
 	return machineClient, nil
 }
 
-func (cli *CLI) AddMachine(ctx context.Context, remoteMachine RemoteMachine, clusterName, machineName string) error {
+// TODO:
+func (cli *CLI) AddMachine(
+	ctx context.Context, remoteMachine RemoteMachine, clusterName, machineName string,
+) (*client.Client, error) {
 	c, err := cli.ConnectCluster(ctx, clusterName)
 	if err != nil {
-		return fmt.Errorf("connect to cluster: %w", err)
+		return nil, fmt.Errorf("connect to cluster: %w", err)
 	}
-	defer func() {
+	defer c.Close()
-		_ = c.Close()
-	}()
 
 	machineClient, err := cli.provisionRemoteMachine(ctx, remoteMachine)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	defer machineClient.Close()
+	defer func() {
+		if err != nil {
+			machineClient.Close()
+		}
+	}()
 
 	// Check if the machine is already initialised as a cluster member and prompt the user to reset it first.
 	minfo, err := machineClient.Inspect(ctx, &emptypb.Empty{})
 	if err != nil {
-		return fmt.Errorf("inspect machine: %w", err)
+		return nil, fmt.Errorf("inspect machine: %w", err)
 	}
 	if minfo.Id != "" {
 		if err = cli.promptResetMachine(); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
 	tokenResp, err := machineClient.Token(ctx, &emptypb.Empty{})
 	if err != nil {
-		return fmt.Errorf("get remote machine token: %w", err)
+		return nil, fmt.Errorf("get remote machine token: %w", err)
 	}
 	token, err := machine.ParseToken(tokenResp.Token)
 	if err != nil {
-		return fmt.Errorf("parse remote machine token: %w", err)
+		return nil, fmt.Errorf("parse remote machine token: %w", err)
 	}
 
 	// Register the machine in the cluster using its public key and endpoints from the token.
@@ -234,13 +239,13 @@ func (cli *CLI) AddMachine(ctx context.Context, remoteMachine RemoteMachine, clu
 	}
 	addResp, err := c.AddMachine(ctx, addReq)
 	if err != nil {
-		return fmt.Errorf("add machine to cluster: %w", err)
+		return nil, fmt.Errorf("add machine to cluster: %w", err)
 	}
 
 	// List other machines in the cluster to include them in the join request.
 	machines, err := c.ListMachines(ctx)
 	if err != nil {
-		return fmt.Errorf("list cluster machines: %w", err)
+		return nil, fmt.Errorf("list cluster machines: %w", err)
 	}
 	otherMachines := make([]*pb.MachineInfo, 0, len(machines)-1)
 	for _, m := range machines {
@@ -255,7 +260,7 @@ func (cli *CLI) AddMachine(ctx context.Context, remoteMachine RemoteMachine, clu
 		OtherMachines: otherMachines,
 	}
 	if _, err = machineClient.JoinCluster(ctx, joinReq); err != nil {
-		return fmt.Errorf("join cluster: %w", err)
+		return nil, fmt.Errorf("join cluster: %w", err)
 	}
 
 	fmt.Printf("Machine %q added to cluster\n", addResp.Machine.Name)
@@ -270,10 +275,10 @@ func (cli *CLI) AddMachine(ctx context.Context, remoteMachine RemoteMachine, clu
 	}
 	cli.config.Clusters[clusterName].Connections = append(cli.config.Clusters[clusterName].Connections, connCfg)
 	if err = cli.config.Save(); err != nil {
-		return fmt.Errorf("save config: %w", err)
+		return nil, fmt.Errorf("save config: %w", err)
 	}
 
-	return nil
+	return machineClient, nil
 }
 
 // provisionRemoteMachine installs the Uncloud daemon and dependencies on the remote machine over SSH and returns

internal/machine/cluster/cluster.go

@@ -232,24 +232,3 @@ func (c *Cluster) ListMachines(ctx context.Context, _ *emptypb.Empty) (*pb.ListM
 
 	return &pb.ListMachinesResponse{Machines: members}, nil
 }
-
-//func (c *Cluster) ListServices(ctx context.Context, _ *emptypb.Empty) (*pb.ListServicesResponse, error) {
-//	if err := c.checkInitialised(ctx); err != nil {
-//		return nil, err
-//	}
-//
-//	return &pb.ListServicesResponse{
-//		Messages: []*pb.Services{
-//			{
-//				Services: []*pb.Service{
-//					{
-//						Name: "service1",
-//					},
-//					{
-//						Name: "service2",
-//					},
-//				},
-//			},
-//		},
-//	}, nil
-//}