diff --git a/go.mod b/go.mod index bd2cd733..dd2af3ce 100644 --- a/go.mod +++ b/go.mod @@ -66,6 +66,7 @@ require ( google.golang.org/grpc v1.74.2 google.golang.org/protobuf v1.36.9 modernc.org/sqlite v1.36.3 + tags.cncf.io/container-device-interface v1.0.1 ) require ( @@ -415,5 +416,4 @@ require ( sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect sigs.k8s.io/yaml v1.4.0 // indirect - tags.cncf.io/container-device-interface v1.0.1 // indirect ) diff --git a/internal/machine/docker/server.go b/internal/machine/docker/server.go index 9b44f5af..2f74875e 100644 --- a/internal/machine/docker/server.go +++ b/internal/machine/docker/server.go @@ -617,7 +617,7 @@ func (s *Server) CreateServiceContainer( NanoCPUs: spec.Container.Resources.CPU, Memory: spec.Container.Resources.Memory, MemoryReservation: spec.Container.Resources.MemoryReservation, - Devices: spec.Container.Resources.DeviceMappings, + Devices: toDockerDevices(spec.Container.Resources.Devices), DeviceRequests: spec.Container.Resources.DeviceReservations, Ulimits: toDockerUlimits(spec.Container.Resources.Ulimits), }, @@ -897,6 +897,23 @@ func toDockerUlimits(ulimits map[string]api.Ulimit) []*units.Ulimit { return dockerUlimits } +func toDockerDevices(devices []api.DeviceMapping) []container.DeviceMapping { + if len(devices) == 0 { + return nil + } + + dockerDevices := make([]container.DeviceMapping, 0, len(devices)) + for _, d := range devices { + dockerDevices = append(dockerDevices, container.DeviceMapping{ + PathOnHost: d.HostPath, + PathInContainer: d.ContainerPath, + CgroupPermissions: d.CgroupPermissions, + }) + } + + return dockerDevices +} + // verifyDockerVolumesExist checks if the Docker named volumes referenced in the mounts exist on the machine. func (s *Server) verifyDockerVolumesExist(ctx context.Context, mounts []mount.Mount) error { for _, m := range mounts { diff --git a/pkg/api/resources.go b/pkg/api/resources.go index f077a635..15bf9e5b 100644 --- a/pkg/api/resources.go +++ b/pkg/api/resources.go @@ -17,14 +17,24 @@ type ContainerResources struct { // MemoryReservation is the minimum amount of memory (in bytes) the container needs to run efficiently. // TODO: implement a placement constraint that checks available memory on machines. MemoryReservation int64 - // Device mappings for direct access to host devices - DeviceMappings []container.DeviceMapping - // Device reservations/requests for access to things like GPUs + // Devices provides direct access to host devices. + Devices []DeviceMapping + // DeviceReservations requests for access to things like GPUs. DeviceReservations []container.DeviceRequest // Ulimits defines the resource limits for the container. Ulimits map[string]Ulimit } +// DeviceMapping represents a device mapping between host and container. +type DeviceMapping struct { + // HostPath is the path to the device on the host. + HostPath string + // ContainerPath is the path to the device in the container. + ContainerPath string + // CgroupPermissions is the cgroup permissions for the device (e.g., "rwm"). + CgroupPermissions string +} + type Ulimit struct { Soft int64 Hard int64 diff --git a/pkg/api/service.go b/pkg/api/service.go index bcd544c8..09c2561e 100644 --- a/pkg/api/service.go +++ b/pkg/api/service.go @@ -383,8 +383,8 @@ func (s *ContainerSpec) Clone() ContainerSpec { if s.Resources.Ulimits != nil { spec.Resources.Ulimits = maps.Clone(s.Resources.Ulimits) } - if s.Resources.DeviceMappings != nil { - spec.Resources.DeviceMappings = slices.Clone(s.Resources.DeviceMappings) + if s.Resources.Devices != nil { + spec.Resources.Devices = slices.Clone(s.Resources.Devices) } if s.Resources.DeviceReservations != nil { spec.Resources.DeviceReservations = slices.Clone(s.Resources.DeviceReservations) diff --git a/pkg/api/service_test.go b/pkg/api/service_test.go index 8a4f2144..da8fd65f 100644 --- a/pkg/api/service_test.go +++ b/pkg/api/service_test.go @@ -232,8 +232,8 @@ func TestContainerSpec_Clone(t *testing.T) { CPU: 1234, Memory: 2345, MemoryReservation: 3456, - DeviceMappings: []container.DeviceMapping{ - {PathOnHost: "/dev/sda", PathInContainer: "/dev/xvda", CgroupPermissions: "rwm"}, + Devices: []DeviceMapping{ + {HostPath: "/dev/sda", ContainerPath: "/dev/xvda", CgroupPermissions: "rwm"}, }, DeviceReservations: []container.DeviceRequest{ {Count: 1, Capabilities: [][]string{{"gpu"}}, Driver: "nvidia"}, @@ -270,7 +270,7 @@ func TestContainerSpec_Clone(t *testing.T) { original.ConfigMounts[0].ContainerPath = stringModified *original.ConfigMounts[0].Mode = 0o755 // Modify the Mode pointer value original.Sysctls["net.ipv4.ip_forward"] = stringModified - original.Resources.DeviceMappings[0].PathOnHost = stringModified + original.Resources.Devices[0].HostPath = stringModified original.Resources.DeviceReservations[0].Count = 2 original.Resources.DeviceReservations[0].Driver = stringModified @@ -293,7 +293,7 @@ func TestContainerSpec_Clone(t *testing.T) { assert.Equal(t, int64(1234), cloned.Resources.CPU) assert.Equal(t, int64(2345), cloned.Resources.Memory) assert.Equal(t, int64(3456), cloned.Resources.MemoryReservation) - assert.Equal(t, "/dev/sda", cloned.Resources.DeviceMappings[0].PathOnHost) + assert.Equal(t, "/dev/sda", cloned.Resources.Devices[0].HostPath) assert.Equal(t, 1, cloned.Resources.DeviceReservations[0].Count) assert.Equal(t, "nvidia", cloned.Resources.DeviceReservations[0].Driver) assert.Equal(t, "1000:1000", cloned.User) diff --git a/pkg/client/compose/service.go b/pkg/client/compose/service.go index d1983c11..2a33a9a5 100644 --- a/pkg/client/compose/service.go +++ b/pkg/client/compose/service.go @@ -11,6 +11,7 @@ import ( "github.com/docker/docker/api/types/mount" "github.com/opencontainers/go-digest" "github.com/psviderski/uncloud/pkg/api" + cdi "tags.cncf.io/container-device-interface/pkg/parser" ) func ServiceSpecFromCompose(project *types.Project, serviceName string) (api.ServiceSpec, error) { @@ -140,7 +141,27 @@ func resourcesFromCompose(service types.ServiceConfig) api.ContainerResources { Memory: int64(service.MemLimit), MemoryReservation: int64(service.MemReservation), Ulimits: ulimitsFromCompose(service.Ulimits), - DeviceMappings: devicesFromCompose(service.Devices), + } + + // Convert device mappings, separating CDI devices from regular device mappings. + // CDI devices are identified when Source == Target and the source is a qualified CDI name. + var cdiDeviceNames []string + for _, dev := range service.Devices { + if dev.Source == dev.Target && cdi.IsQualifiedName(dev.Source) { + cdiDeviceNames = append(cdiDeviceNames, dev.Source) + continue + } + resources.Devices = append(resources.Devices, api.DeviceMapping{ + HostPath: dev.Source, + ContainerPath: dev.Target, + CgroupPermissions: dev.Permissions, + }) + } + if len(cdiDeviceNames) > 0 { + resources.DeviceReservations = append(resources.DeviceReservations, container.DeviceRequest{ + Driver: "cdi", + DeviceIDs: cdiDeviceNames, + }) } // Convert GPU device requests from compose format, appending "gpu" capability. @@ -336,28 +357,6 @@ func ulimitsFromCompose(ulimits map[string]*types.UlimitsConfig) map[string]api. return res } -func devicesFromCompose(composeDevices []types.DeviceMapping) []container.DeviceMapping { - mappings := make([]container.DeviceMapping, 0, len(composeDevices)) - - for _, dev := range composeDevices { - mapping := container.DeviceMapping{ - PathOnHost: dev.Source, - PathInContainer: dev.Target, - CgroupPermissions: dev.Permissions, - } - - if mapping.PathInContainer == "" { - mapping.PathInContainer = mapping.PathOnHost - } - if mapping.CgroupPermissions == "" { - mapping.CgroupPermissions = "rwm" - } - - mappings = append(mappings, mapping) - } - return mappings -} - // validateServicesExtensions validates extension combinations across all services in the project. func validateServicesExtensions(project *types.Project) error { for _, service := range project.Services { diff --git a/pkg/client/compose/service_test.go b/pkg/client/compose/service_test.go index 02e1fb40..e746d605 100644 --- a/pkg/client/compose/service_test.go +++ b/pkg/client/compose/service_test.go @@ -1053,12 +1053,13 @@ services: func TestServiceSpecFromCompose_Devices(t *testing.T) { tests := []struct { - name string - composeYAML string - expectedMappings []container.DeviceMapping + name string + composeYAML string + expectedDevices []api.DeviceMapping + expectedReservations []container.DeviceRequest }{ { - name: "devices_simple", + name: "simple device", composeYAML: ` services: test: @@ -1066,16 +1067,12 @@ services: devices: - /dev/dri `, - expectedMappings: []container.DeviceMapping{ - { - PathOnHost: "/dev/dri", - PathInContainer: "/dev/dri", - CgroupPermissions: "rwm", - }, + expectedDevices: []api.DeviceMapping{ + {HostPath: "/dev/dri", ContainerPath: "/dev/dri", CgroupPermissions: "rwm"}, }, }, { - name: "devices_full", + name: "device with target and permissions", composeYAML: ` services: test: @@ -1083,16 +1080,12 @@ services: devices: - /dev/sda:/dev/xvda:r `, - expectedMappings: []container.DeviceMapping{ - { - PathOnHost: "/dev/sda", - PathInContainer: "/dev/xvda", - CgroupPermissions: "r", - }, + expectedDevices: []api.DeviceMapping{ + {HostPath: "/dev/sda", ContainerPath: "/dev/xvda", CgroupPermissions: "r"}, }, }, { - name: "multiple_devices", + name: "multiple devices", composeYAML: ` services: test: @@ -1102,22 +1095,43 @@ services: - /dev/sda:/dev/xvda - "/dev/dri" `, - expectedMappings: []container.DeviceMapping{ - { - PathOnHost: "/dev/ttyUSB0", - PathInContainer: "/dev/ttyUSB0", - CgroupPermissions: "rw", - }, - { - PathOnHost: "/dev/sda", - PathInContainer: "/dev/xvda", - CgroupPermissions: "rwm", - }, - { - PathOnHost: "/dev/dri", - PathInContainer: "/dev/dri", - CgroupPermissions: "rwm", - }, + expectedDevices: []api.DeviceMapping{ + {HostPath: "/dev/ttyUSB0", ContainerPath: "/dev/ttyUSB0", CgroupPermissions: "rw"}, + {HostPath: "/dev/sda", ContainerPath: "/dev/xvda", CgroupPermissions: "rwm"}, + {HostPath: "/dev/dri", ContainerPath: "/dev/dri", CgroupPermissions: "rwm"}, + }, + }, + { + name: "CDI device", + composeYAML: ` +services: + test: + image: nginx + devices: + - vendor.com/class=device1 +`, + expectedReservations: []container.DeviceRequest{ + {Driver: "cdi", DeviceIDs: []string{"vendor.com/class=device1"}}, + }, + }, + { + name: "mixed CDI and regular devices", + composeYAML: ` +services: + test: + image: nginx + devices: + - /dev/dri + - vendor.com/class=device1 + - nvidia.com/gpu=0 + - /dev/sda:/dev/xvda:r +`, + expectedDevices: []api.DeviceMapping{ + {HostPath: "/dev/dri", ContainerPath: "/dev/dri", CgroupPermissions: "rwm"}, + {HostPath: "/dev/sda", ContainerPath: "/dev/xvda", CgroupPermissions: "r"}, + }, + expectedReservations: []container.DeviceRequest{ + {Driver: "cdi", DeviceIDs: []string{"vendor.com/class=device1", "nvidia.com/gpu=0"}}, }, }, } @@ -1130,8 +1144,8 @@ services: spec, err := ServiceSpecFromCompose(project, "test") require.NoError(t, err) - assert.Equal(t, tt.expectedMappings, spec.Container.Resources.DeviceMappings, - "DeviceMappings should match expected") + assert.Equal(t, tt.expectedDevices, spec.Container.Resources.Devices) + assert.Equal(t, tt.expectedReservations, spec.Container.Resources.DeviceReservations) }) } } diff --git a/pkg/client/deploy/container.go b/pkg/client/deploy/container.go index eb1b72ac..ca97ac9c 100644 --- a/pkg/client/deploy/container.go +++ b/pkg/client/deploy/container.go @@ -87,7 +87,7 @@ func EvalContainerSpecChange(current api.ServiceSpec, new api.ServiceSpec) Conta if !reflect.DeepEqual(current.Container.Resources.DeviceReservations, newResources.DeviceReservations) { return ContainerNeedsRecreate } - if !reflect.DeepEqual(current.Container.Resources.DeviceMappings, newResources.DeviceMappings) { + if !reflect.DeepEqual(current.Container.Resources.Devices, newResources.Devices) { return ContainerNeedsRecreate } diff --git a/pkg/client/deploy/container_test.go b/pkg/client/deploy/container_test.go index 1ba53340..cc99c933 100644 --- a/pkg/client/deploy/container_test.go +++ b/pkg/client/deploy/container_test.go @@ -1427,26 +1427,26 @@ func TestEvalContainerSpecChange_DeviceMappings(t *testing.T) { }, { name: "identical mapping", - current: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/foo", CgroupPermissions: "rwm"}}}, - new: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/foo", CgroupPermissions: "rwm"}}}, + current: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/foo", CgroupPermissions: "rwm"}}}, + new: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/foo", CgroupPermissions: "rwm"}}}, want: ContainerUpToDate, }, { name: "add mapping", current: api.ContainerResources{}, - new: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/foo", CgroupPermissions: "rwm"}}}, + new: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/foo", CgroupPermissions: "rwm"}}}, want: ContainerNeedsRecreate, }, { name: "remove mapping", - current: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/foo", CgroupPermissions: "rwm"}}}, + current: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/foo", CgroupPermissions: "rwm"}}}, new: api.ContainerResources{}, want: ContainerNeedsRecreate, }, { name: "change mapping path", - current: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/foo", CgroupPermissions: "rwm"}}}, - new: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/bar", CgroupPermissions: "rwm"}}}, + current: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/foo", CgroupPermissions: "rwm"}}}, + new: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/bar", CgroupPermissions: "rwm"}}}, want: ContainerNeedsRecreate, }, }