feat: Implement basic build & push capabilities for services (#68)

* wip: Build experiments

* wip: overall structure for build/push

* ref: Split method, add test fixture

* feat: Basic push functionality

* ref: Better structure and output

* feat: Handle registry auth

* feat: Integrate with deploy

* ref: Split into files

* fix: Modernize push options

cmd/uncloud/build.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	composecli "github.com/compose-spec/compose-go/v2/cli"
+	"github.com/psviderski/uncloud/internal/cli"
+	"github.com/psviderski/uncloud/pkg/client/compose"
+	"github.com/spf13/cobra"
+)
+
+// NewBuildCommand creates a new command to build services from a Compose file.
+func NewBuildCommand() *cobra.Command {
+	opts := cli.BuildOptions{}
+	cmd := &cobra.Command{
+		Use:   "build [FLAGS] [SERVICE...]",
+		Short: "Build services from a Compose file.",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			uncli := cmd.Context().Value("cli").(*cli.CLI)
+
+			if len(args) > 0 {
+				opts.Services = args
+			}
+
+			return runBuild(cmd.Context(), uncli, opts)
+		},
+	}
+
+	cmd.Flags().StringSliceVarP(&opts.Files, "file", "f", nil,
+		"One or more Compose files to build (default compose.yaml)")
+	cmd.Flags().StringSliceVarP(&opts.Profiles, "profile", "p", nil,
+		"One or more Compose profiles to enable.")
+	cmd.Flags().BoolVarP(&opts.Push, "push", "P", false,
+		"Push built images to the registry after building. (default false)")
+	cmd.Flags().BoolVarP(&opts.NoCache, "no-cache", "n", false,
+		"Do not use cache when building images. (default false)")
+
+	return cmd
+}
+
+// TODO: deduplicate with a similar functino for deploy options
+// projectOpts returns the project options for the Compose file(s).
+func projectOptsFromBuildOpts(opts cli.BuildOptions) []composecli.ProjectOptionsFn {
+	projectOpts := []composecli.ProjectOptionsFn{}
+
+	if len(opts.Profiles) > 0 {
+		projectOpts = append(projectOpts, composecli.WithDefaultProfiles(opts.Profiles...))
+	}
+
+	return projectOpts
+}
+
+// runBuild parses the Compose file(s), builds the services, and pushes them if requested.
+func runBuild(ctx context.Context, uncli *cli.CLI, opts cli.BuildOptions) error {
+	projectOpts := projectOptsFromBuildOpts(opts)
+	project, err := compose.LoadProject(ctx, opts.Files, projectOpts...)
+	if err != nil {
+		return fmt.Errorf("load compose file(s): %w", err)
+	}
+
+	if len(opts.Services) > 0 {
+		project, err = project.WithSelectedServices(opts.Services)
+		if err != nil {
+			return fmt.Errorf("select services: %w", err)
+		}
+	}
+
+	servicesToBuild := cli.GetServicesThatNeedBuild(project)
+
+	if len(servicesToBuild) == 0 {
+		fmt.Println("No services to build.")
+		return nil
+	}
+
+	return cli.BuildServices(ctx, servicesToBuild, opts)
+}

cmd/uncloud/deploy.go

@@ -20,6 +20,7 @@ type deployOptions struct {
 	files    []string
 	profiles []string
 	services []string
+	noBuild  bool
 
 	context string
 }
@@ -47,6 +48,9 @@ func NewDeployCommand() *cobra.Command {
 		"One or more Compose profiles to enable.")
 	cmd.Flags().StringVarP(&opts.context, "context", "c", "",
 		"Name of the cluster context to deploy to (default is the current context)")
+	cmd.Flags().BoolVarP(&opts.noBuild, "no-build", "n", false,
+		"Do not build images before deploying services. (default false)")
+
 	// TODO: Consider adding a filter flag to specify which machines to deploy to but keep the rest running.
 	//  Could be useful to test a new version on a subset of machines before rolling out to all.
 
@@ -81,6 +85,26 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
 		}
 	}
 
+	servicesToBuild := cli.GetServicesThatNeedBuild(project)
+
+	if len(servicesToBuild) > 0 {
+		if opts.noBuild {
+			fmt.Println("Not building services as requested.")
+		} else {
+			buildOpts := cli.BuildOptions{
+				Files:    opts.files,
+				Profiles: opts.profiles,
+				Services: opts.services,
+				Push:     true,
+				NoCache:  false,
+			}
+
+			if err := cli.BuildServices(ctx, servicesToBuild, buildOpts); err != nil {
+				return fmt.Errorf("build services: %w", err)
+			}
+		}
+	}
+
 	clusterClient, err := uncli.ConnectCluster(ctx, opts.context)
 	if err != nil {
 		return fmt.Errorf("connect to cluster: %w", err)

cmd/uncloud/main.go

@@ -75,6 +75,7 @@ func main() {
 
 	cmd.AddCommand(
 		NewDeployCommand(),
+		NewBuildCommand(),
 		caddy.NewRootCommand(),
 		cmdcontext.NewRootCommand(),
 		dns.NewRootCommand(),

go.mod

@@ -213,6 +213,8 @@ require (
 	github.com/moby/patternmatcher v0.6.0 // indirect
 	github.com/moby/sys/sequential v0.6.0 // indirect
 	github.com/moby/sys/signal v0.7.1 // indirect
+	github.com/moby/sys/user v0.3.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect

internal/cli/build.go

@@ -0,0 +1,197 @@
+package cli
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	composetypes "github.com/compose-spec/compose-go/v2/types"
+	"github.com/distribution/reference"
+	"github.com/docker/cli/cli/config"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/image"
+	dockerclient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/registry"
+)
+
+type BuildOptions struct {
+	Files    []string
+	Profiles []string
+	Services []string
+	Push     bool
+	NoCache  bool
+}
+
+// GetServicesThatNeedBuild returns a map of services that require building
+func GetServicesThatNeedBuild(project *composetypes.Project) map[string]composetypes.ServiceConfig {
+	servicesToBuild := make(map[string]composetypes.ServiceConfig, len(project.Services))
+	for serviceName, service := range project.Services {
+		if service.Build == nil {
+			continue
+		}
+		servicesToBuild[serviceName] = service
+	}
+	return servicesToBuild
+}
+
+// BuildServices builds the services defined in the provided map.
+func BuildServices(ctx context.Context, servicesToBuild map[string]composetypes.ServiceConfig, opts BuildOptions) error {
+	fmt.Println("Building services...")
+
+	// Init docker client (can be local or remote, depending on DOCKER_HOST environment variable)
+	dockerCli, err := dockerclient.NewClientWithOpts(dockerclient.FromEnv, dockerclient.WithAPIVersionNegotiation())
+	if err != nil {
+		return err
+	}
+	defer dockerCli.Close()
+
+	serviceImages := make(map[string]string, len(servicesToBuild))
+
+	// Build the services using the local docker client and compose libraries
+	for _, service := range servicesToBuild {
+		fmt.Printf("Building service: %s\n", service.Name)
+		imageName, err := buildSingleService(ctx, dockerCli, service, opts)
+		if err != nil {
+			return fmt.Errorf("build service %s: %w", service.Name, err)
+		}
+		serviceImages[service.Name] = imageName
+	}
+	fmt.Printf("Service images are built.\n")
+
+	if opts.Push {
+		err = pushServiceImages(ctx, dockerCli, serviceImages)
+	}
+
+	return err
+}
+
+// buildSingleService builds a single service using the Docker client and Compose libraries.
+func buildSingleService(ctx context.Context, dockerCli *dockerclient.Client, service composetypes.ServiceConfig, opts BuildOptions) (string, error) {
+	if service.Build == nil {
+		return "", fmt.Errorf("service %s has no build configuration", service.Name)
+	}
+	if service.Image == "" {
+		return "", fmt.Errorf("service %s has no image specified; building services without image is not supported yet", service.Name)
+	}
+
+	buildContextPath := service.Build.Context
+	imageName := service.Image
+
+	// Create a tar archive of the build context
+	buildContext, err := archive.TarWithOptions(buildContextPath, &archive.TarOptions{})
+	if err != nil {
+		return "", fmt.Errorf("failed to create build context for service %s: %w", service.Name, err)
+	}
+
+	buildOptions := types.ImageBuildOptions{
+		// TODO: Support Dockerfiles outside the build context
+		// See https://github.com/docker/compose/blob/cf89fd1aa1328d5af77658ccc5a1e1b29981ae80/pkg/compose/build_classic.go#L92
+		Dockerfile: service.Build.Dockerfile,
+		Tags:       []string{imageName},
+		Remove:     true, // Remove intermediate containers
+		NoCache:    opts.NoCache,
+	}
+
+	buildResponse, err := dockerCli.ImageBuild(ctx, buildContext, buildOptions)
+	if err != nil {
+		return "", fmt.Errorf("failed to build image for service %s: %w", service.Name, err)
+	}
+	defer buildResponse.Body.Close()
+
+	// Print the build output
+	decoder := json.NewDecoder(buildResponse.Body)
+	for {
+		var message map[string]interface{}
+		if err := decoder.Decode(&message); err == io.EOF {
+			break
+		} else if err != nil {
+			return "", fmt.Errorf("failed to decode build output for service %s: %w", service.Name, err)
+		}
+
+		if stream, ok := message["stream"]; ok {
+			fmt.Print(stream)
+		}
+	}
+
+	return imageName, nil
+}
+
+// pushSingleServiceImage pushes a single service image.
+func pushSingleServiceImage(ctx context.Context, dockerCli *dockerclient.Client, serviceName string, imageName string) error {
+	ref, err := reference.ParseNormalizedNamed(imageName)
+	if err != nil {
+		return err
+	}
+
+	repoInfo, err := registry.ParseRepositoryInfo(ref)
+	if err != nil {
+		return err
+	}
+
+	registryKey := repoInfo.Index.Name
+	if repoInfo.Index.Official {
+		registryKey = registry.IndexServer
+	}
+
+	// Load the Docker config file with auth details, if available
+	configFile := config.LoadDefaultConfigFile(os.Stderr)
+
+	authConfig, err := configFile.GetAuthConfig(registryKey)
+	if err != nil {
+		return err
+	}
+
+	authJSON, err := json.Marshal(authConfig)
+	if err != nil {
+		return fmt.Errorf("failed to marshal auth config for registry %s: %w", registryKey, err)
+	}
+
+	authStr := base64.URLEncoding.EncodeToString(authJSON)
+
+	pushOptions := image.PushOptions{
+		RegistryAuth: authStr,
+	}
+	pushResponse, err := dockerCli.ImagePush(ctx, imageName, pushOptions)
+	if err != nil {
+		return fmt.Errorf("failed to push image %s: %w", imageName, err)
+	}
+	defer pushResponse.Close()
+
+	fmt.Printf("Pushing image %s for service %s\n", imageName, serviceName)
+
+	// Handle output and errors
+	decoder := json.NewDecoder(pushResponse)
+	for {
+		var message map[string]interface{}
+		if err := decoder.Decode(&message); err == io.EOF {
+			break
+		} else if err != nil {
+			return fmt.Errorf("failed to decode push output for image %s: %w", imageName, err)
+		}
+		if stream, ok := message["stream"]; ok {
+			fmt.Print(stream)
+		} else if errorMessage, ok := message["error"]; ok {
+			return fmt.Errorf("error pushing image %s: %s", imageName, errorMessage)
+		} else if status, ok := message["status"]; ok {
+			fmt.Printf("   %s\n", status)
+		}
+	}
+
+	fmt.Printf("Image %s pushed successfully.\n", imageName)
+	return nil
+}
+
+// pushServiceImages pushes all built service images to the registry.
+func pushServiceImages(ctx context.Context, dockerCli *dockerclient.Client, serviceImages map[string]string) error {
+	fmt.Printf("Pushing images...\n")
+	for serviceName, imageName := range serviceImages {
+		if err := pushSingleServiceImage(ctx, dockerCli, serviceName, imageName); err != nil {
+			return fmt.Errorf("push image for service %s: %w", serviceName, err)
+		}
+	}
+	return nil
+}

test/e2e/fixtures/compose-build-basic/busybox-first/Dockerfile

@@ -0,0 +1,3 @@
+FROM busybox:1.37.0-musl
+
+ENV SERVICE_NAME=busybox-first

test/e2e/fixtures/compose-build-basic/busybox-second/Dockerfile.alt

@@ -0,0 +1,3 @@
+FROM busybox:1.37.0-musl
+
+ENV SERVICE_NAME=busybox-second

test/e2e/fixtures/compose-build-basic/compose.yaml

@@ -0,0 +1,14 @@
+services:
+  service-no-build:
+    image: portainer/pause:3.9
+
+  busybox-first:
+    image: localhost:5000/busybox-first
+    build:
+      context: busybox-first/
+
+  busybox-second:
+    image: localhost:5000/busybox-second
+    build:
+      context: busybox-second/
+      dockerfile: Dockerfile.alt