From ecbd4378e459ae5466254ab81e476ff7e5814318 Mon Sep 17 00:00:00 2001 From: Pavel Sviderski Date: Sat, 29 Mar 2025 15:14:43 +1000 Subject: [PATCH] feat(scale): store container spec in local machine.db sqlite --- cmd/uncloud/main.go | 5 ++-- go.mod | 9 +++++-- go.sum | 29 +++++++++++++++++++++ internal/machine/db.go | 42 +++++++++++++++++++++++++++++++ internal/machine/docker/server.go | 34 +++++++++++++++++++++---- internal/machine/machine.go | 25 +++++++++++------- 6 files changed, 126 insertions(+), 18 deletions(-) create mode 100644 internal/machine/db.go diff --git a/cmd/uncloud/main.go b/cmd/uncloud/main.go index 0e904a81..4217b1af 100644 --- a/cmd/uncloud/main.go +++ b/cmd/uncloud/main.go @@ -3,6 +3,9 @@ package main import ( "context" "fmt" + "net/netip" + "strings" + "github.com/psviderski/uncloud/cmd/uncloud/caddy" "github.com/psviderski/uncloud/cmd/uncloud/dns" "github.com/psviderski/uncloud/cmd/uncloud/machine" @@ -11,8 +14,6 @@ import ( "github.com/psviderski/uncloud/internal/cli/config" "github.com/psviderski/uncloud/internal/fs" "github.com/spf13/cobra" - "net/netip" - "strings" ) type globalOptions struct { diff --git a/go.mod b/go.mod index 91e29870..48162c34 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/ipfs/go-ds-crdt v0.5.2 github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 + github.com/jmoiron/sqlx v1.4.0 github.com/lmittmann/tint v1.0.5 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0 @@ -51,6 +52,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 google.golang.org/grpc v1.68.1 google.golang.org/protobuf v1.36.3 + modernc.org/sqlite v1.36.3 ) require ( @@ -172,13 +174,11 @@ require ( github.com/jackc/pgtype v1.14.0 // indirect github.com/jackc/pgx/v4 v4.18.3 // indirect github.com/jbenet/goprocess v0.1.4 // indirect - github.com/jmoiron/sqlx v1.4.0 // indirect github.com/josharian/native v1.1.0 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect - github.com/lib/pq v1.10.9 // indirect github.com/libdns/libdns v0.2.2 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-libp2p v0.35.4 // indirect @@ -228,6 +228,7 @@ require ( github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/onsi/ginkgo/v2 v2.22.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pires/go-proxyproto v0.7.0 // indirect @@ -242,6 +243,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.48.2 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rs/xid v1.5.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -312,4 +314,7 @@ require ( gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 // indirect howett.net/plist v1.0.0 // indirect lukechampine.com/blake3 v1.3.0 // indirect + modernc.org/libc v1.61.13 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.8.2 // indirect ) diff --git a/go.sum b/go.sum index d815c52a..ac62e5ce 100644 --- a/go.sum +++ b/go.sum @@ -704,6 +704,7 @@ github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.6.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw= @@ -813,6 +814,8 @@ github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOEL github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= @@ -929,6 +932,8 @@ github.com/quic-go/webtransport-go v0.8.0 h1:HxSrwun11U+LlmwpgM1kEqIqH90IT4N8auv github.com/quic-go/webtransport-go v0.8.0/go.mod h1:N99tjprW432Ut5ONql/aUhSLT0YVSlwHohQsuac9WaM= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -1431,3 +1436,27 @@ k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0= +modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.23.16 h1:Z2N+kk38b7SfySC1ZkpGLN2vthNJP1+ZzGZIlH7uBxo= +modernc.org/ccgo/v4 v4.23.16/go.mod h1:nNma8goMTY7aQZQNTyN9AIoJfxav4nvTnvKThAeMDdo= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.6.3 h1:aJVhcqAte49LF+mGveZ5KPlsp4tdGdAOT4sipJXADjw= +modernc.org/gc/v2 v2.6.3/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/libc v1.61.13 h1:3LRd6ZO1ezsFiX1y+bHd1ipyEHIJKvuprv0sLTBwLW8= +modernc.org/libc v1.61.13/go.mod h1:8F/uJWL/3nNil0Lgt1Dpz+GgkApWh04N3el3hxJcA6E= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.8.2 h1:cL9L4bcoAObu4NkxOlKWBWtNHIsnnACGF/TbqQ6sbcI= +modernc.org/memory v1.8.2/go.mod h1:ZbjSvMO5NQ1A2i3bWeDiVMxIorXwdClKE/0SZ+BMotU= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.36.3 h1:qYMYlFR+rtLDUzuXoST1SDIdEPbX8xzuhdF90WsX1ss= +modernc.org/sqlite v1.36.3/go.mod h1:ADySlx7K4FdY5MaJcEv86hTJ0PjedAloTUuif0YS3ws= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/machine/db.go b/internal/machine/db.go new file mode 100644 index 00000000..055d5a18 --- /dev/null +++ b/internal/machine/db.go @@ -0,0 +1,42 @@ +package machine + +import ( + "fmt" + + "github.com/jmoiron/sqlx" + _ "modernc.org/sqlite" +) + +const DBFileName = "machine.db" + +// NewDB creates a new connection to machine SQLite database and runs schema migrations if necessary. +func NewDB(path string) (*sqlx.DB, error) { + // - Write-Ahead Logging (WAL) mode for better read/write performance. + // - Busy timeout (5s) to make concurrent writes wait on each other instead of failing immediately. + conn := path + "?_pragma=journal_mode=WAL&_pragma=synchronous=NORMAL&_pragma=busy_timeout=5000&_time_format=sqlite" + db, err := sqlx.Connect("sqlite", conn) + if err != nil { + return nil, fmt.Errorf("connect to SQLite database '%s': %w", conn, err) + } + + schema := ` + CREATE TABLE IF NOT EXISTS containers ( + id TEXT NOT NULL PRIMARY KEY, + service_id TEXT NOT NULL, + service_name TEXT AS (json_extract(service_spec, '$.Name')), + service_spec TEXT NOT NULL CHECK (json_valid(service_spec)), + -- 'subsecond' modifier is used to store timestamps with millisecond precision. + created_at TIMESTAMP NOT NULL DEFAULT (datetime('subsecond')), + updated_at TIMESTAMP NOT NULL DEFAULT (datetime('subsecond')) + ); + + CREATE INDEX IF NOT EXISTS idx_containers_service_id ON containers (service_id); + CREATE INDEX IF NOT EXISTS idx_containers_service_name ON containers (service_name); + ` + + if _, err = db.Exec(schema); err != nil { + return nil, fmt.Errorf("create schema: %w", err) + } + + return db, nil +} diff --git a/internal/machine/docker/server.go b/internal/machine/docker/server.go index 46f2f903..5d58552f 100644 --- a/internal/machine/docker/server.go +++ b/internal/machine/docker/server.go @@ -5,6 +5,10 @@ import ( "encoding/json" "errors" "fmt" + "io" + "strconv" + "strings" + "github.com/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -16,6 +20,7 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/jmoiron/sqlx" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/psviderski/uncloud/internal/machine/api/pb" @@ -25,20 +30,21 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" - "io" - "strconv" - "strings" ) // Server implements the gRPC Docker service that proxies requests to the Docker daemon. type Server struct { pb.UnimplementedDockerServer client *client.Client + db *sqlx.DB } // NewServer creates a new Docker gRPC server with the provided Docker client. -func NewServer(cli *client.Client) *Server { - return &Server{client: cli} +func NewServer(cli *client.Client, db *sqlx.DB) *Server { + return &Server{ + client: cli, + db: db, + } } // CreateContainer creates a new container based on the given configuration. @@ -424,5 +430,23 @@ func (s *Server) CreateServiceContainer( return nil, status.Errorf(codes.Internal, "marshal response: %v", err) } + // Store the container spec in the database or remove the container with its anonymous volumes if storing fails. + removeContainer := func() { + _ = s.client.ContainerRemove(ctx, resp.ID, container.RemoveOptions{RemoveVolumes: true}) + } + + specBytes, err := json.Marshal(spec) + if err != nil { + removeContainer() + return nil, status.Errorf(codes.Internal, "marshal service spec: %v", err) + } + + if _, err = s.db.ExecContext(ctx, + `INSERT INTO containers (id, service_id, service_spec) VALUES ($1, $2, $3)`, + resp.ID, req.ServiceId, string(specBytes)); err != nil { + removeContainer() + return nil, status.Errorf(codes.Internal, "store container in database: %v", err) + } + return &pb.CreateContainerResponse{Response: respBytes}, nil } diff --git a/internal/machine/machine.go b/internal/machine/machine.go index 93ffc743..fc519f74 100644 --- a/internal/machine/machine.go +++ b/internal/machine/machine.go @@ -5,14 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/docker/docker/client" - "github.com/docker/go-connections/sockets" - "github.com/siderolabs/grpc-proxy/proxy" - "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/emptypb" "log/slog" "net" "net/netip" @@ -21,6 +13,9 @@ import ( "path/filepath" "slices" "strconv" + + "github.com/docker/docker/client" + "github.com/docker/go-connections/sockets" "github.com/psviderski/uncloud/internal/corrosion" "github.com/psviderski/uncloud/internal/docker" "github.com/psviderski/uncloud/internal/fs" @@ -32,6 +27,12 @@ import ( machinedocker "github.com/psviderski/uncloud/internal/machine/docker" "github.com/psviderski/uncloud/internal/machine/network" "github.com/psviderski/uncloud/internal/machine/store" + "github.com/siderolabs/grpc-proxy/proxy" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" ) const ( @@ -209,7 +210,13 @@ func NewMachine(config *Config) (*Machine, error) { if err != nil { return nil, fmt.Errorf("create Docker client: %w", err) } - dockerServer := machinedocker.NewServer(dockerCli) + + dbFilePath := filepath.Join(config.DataDir, DBFileName) + db, err := NewDB(dbFilePath) + if err != nil { + return nil, fmt.Errorf("init machine database: %w", err) + } + dockerServer := machinedocker.NewServer(dockerCli, db) // Init a local gRPC proxy server that proxies requests to the local or remote machine API servers. proxyDirector := apiproxy.NewDirector(config.MachineSockPath, APIPort)