philipp/uncloud
1
0
Fork 0
mirror of https://github.com/psviderski/uncloud.git synced 2026-08-26 11:03:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: refactor device mapping to be compliant with Compose (CDI requests)

Browse Source
This commit is contained in:
Pasha Sviderski
2026-02-11 17:44:38 +00:00
parent ab6f856987
commit 5c54e9d059
9 changed files with 117 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1 -1
View File
@@ -66,6 +66,7 @@ require (
google.golang.org/grpc v1.74.2
google.golang.org/protobuf v1.36.9
modernc.org/sqlite v1.36.3
tags.cncf.io/container-device-interface v1.0.1
)
require (
@@ -415,5 +416,4 @@ require (
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
tags.cncf.io/container-device-interface v1.0.1 // indirect
)
internal/machine/docker/server.go
+18 -1
View File
@@ -617,7 +617,7 @@ func (s *Server) CreateServiceContainer(
NanoCPUs: spec.Container.Resources.CPU,
Memory: spec.Container.Resources.Memory,
MemoryReservation: spec.Container.Resources.MemoryReservation,
Devices: spec.Container.Resources.DeviceMappings,
Devices: toDockerDevices(spec.Container.Resources.Devices),
DeviceRequests: spec.Container.Resources.DeviceReservations,
Ulimits: toDockerUlimits(spec.Container.Resources.Ulimits),
},
@@ -897,6 +897,23 @@ func toDockerUlimits(ulimits map[string]api.Ulimit) []*units.Ulimit {
return dockerUlimits
}
func toDockerDevices(devices []api.DeviceMapping) []container.DeviceMapping {
if len(devices) == 0 {
return nil
}
dockerDevices := make([]container.DeviceMapping, 0, len(devices))
for _, d := range devices {
dockerDevices = append(dockerDevices, container.DeviceMapping{
PathOnHost: d.HostPath,
PathInContainer: d.ContainerPath,
CgroupPermissions: d.CgroupPermissions,
})
}
return dockerDevices
}
// verifyDockerVolumesExist checks if the Docker named volumes referenced in the mounts exist on the machine.
func (s *Server) verifyDockerVolumesExist(ctx context.Context, mounts []mount.Mount) error {
for _, m := range mounts {
pkg/api/resources.go
+13 -3
View File
@@ -17,14 +17,24 @@ type ContainerResources struct {
// MemoryReservation is the minimum amount of memory (in bytes) the container needs to run efficiently.
// TODO: implement a placement constraint that checks available memory on machines.
MemoryReservation int64
// Device mappings for direct access to host devices
DeviceMappings []container.DeviceMapping
// Device reservations/requests for access to things like GPUs
// Devices provides direct access to host devices.
Devices []DeviceMapping
// DeviceReservations requests for access to things like GPUs.
DeviceReservations []container.DeviceRequest
// Ulimits defines the resource limits for the container.
Ulimits map[string]Ulimit
}
// DeviceMapping represents a device mapping between host and container.
type DeviceMapping struct {
// HostPath is the path to the device on the host.
HostPath string
// ContainerPath is the path to the device in the container.
ContainerPath string
// CgroupPermissions is the cgroup permissions for the device (e.g., "rwm").
CgroupPermissions string
}
type Ulimit struct {
Soft int64
Hard int64
pkg/api/service.go
+2 -2
View File
@@ -383,8 +383,8 @@ func (s *ContainerSpec) Clone() ContainerSpec {
if s.Resources.Ulimits != nil {
spec.Resources.Ulimits = maps.Clone(s.Resources.Ulimits)
}
if s.Resources.DeviceMappings != nil {
spec.Resources.DeviceMappings = slices.Clone(s.Resources.DeviceMappings)
if s.Resources.Devices != nil {
spec.Resources.Devices = slices.Clone(s.Resources.Devices)
}
if s.Resources.DeviceReservations != nil {
spec.Resources.DeviceReservations = slices.Clone(s.Resources.DeviceReservations)
pkg/api/service_test.go
+4 -4
View File
@@ -232,8 +232,8 @@ func TestContainerSpec_Clone(t *testing.T) {
CPU: 1234,
Memory: 2345,
MemoryReservation: 3456,
DeviceMappings: []container.DeviceMapping{
{PathOnHost: "/dev/sda", PathInContainer: "/dev/xvda", CgroupPermissions: "rwm"},
Devices: []DeviceMapping{
{HostPath: "/dev/sda", ContainerPath: "/dev/xvda", CgroupPermissions: "rwm"},
},
DeviceReservations: []container.DeviceRequest{
{Count: 1, Capabilities: [][]string{{"gpu"}}, Driver: "nvidia"},
@@ -270,7 +270,7 @@ func TestContainerSpec_Clone(t *testing.T) {
original.ConfigMounts[0].ContainerPath = stringModified
*original.ConfigMounts[0].Mode = 0o755 // Modify the Mode pointer value
original.Sysctls["net.ipv4.ip_forward"] = stringModified
original.Resources.DeviceMappings[0].PathOnHost = stringModified
original.Resources.Devices[0].HostPath = stringModified
original.Resources.DeviceReservations[0].Count = 2
original.Resources.DeviceReservations[0].Driver = stringModified
@@ -293,7 +293,7 @@ func TestContainerSpec_Clone(t *testing.T) {
assert.Equal(t, int64(1234), cloned.Resources.CPU)
assert.Equal(t, int64(2345), cloned.Resources.Memory)
assert.Equal(t, int64(3456), cloned.Resources.MemoryReservation)
assert.Equal(t, "/dev/sda", cloned.Resources.DeviceMappings[0].PathOnHost)
assert.Equal(t, "/dev/sda", cloned.Resources.Devices[0].HostPath)
assert.Equal(t, 1, cloned.Resources.DeviceReservations[0].Count)
assert.Equal(t, "nvidia", cloned.Resources.DeviceReservations[0].Driver)
assert.Equal(t, "1000:1000", cloned.User)
pkg/client/compose/service.go
+22 -23
View File
@@ -11,6 +11,7 @@ import (
"github.com/docker/docker/api/types/mount"
"github.com/opencontainers/go-digest"
"github.com/psviderski/uncloud/pkg/api"
cdi "tags.cncf.io/container-device-interface/pkg/parser"
)
func ServiceSpecFromCompose(project *types.Project, serviceName string) (api.ServiceSpec, error) {
@@ -140,7 +141,27 @@ func resourcesFromCompose(service types.ServiceConfig) api.ContainerResources {
Memory: int64(service.MemLimit),
MemoryReservation: int64(service.MemReservation),
Ulimits: ulimitsFromCompose(service.Ulimits),
DeviceMappings: devicesFromCompose(service.Devices),
}
// Convert device mappings, separating CDI devices from regular device mappings.
// CDI devices are identified when Source == Target and the source is a qualified CDI name.
var cdiDeviceNames []string
for _, dev := range service.Devices {
if dev.Source == dev.Target && cdi.IsQualifiedName(dev.Source) {
cdiDeviceNames = append(cdiDeviceNames, dev.Source)
continue
}
resources.Devices = append(resources.Devices, api.DeviceMapping{
HostPath: dev.Source,
ContainerPath: dev.Target,
CgroupPermissions: dev.Permissions,
})
}
if len(cdiDeviceNames) > 0 {
resources.DeviceReservations = append(resources.DeviceReservations, container.DeviceRequest{
Driver: "cdi",
DeviceIDs: cdiDeviceNames,
})
}
// Convert GPU device requests from compose format, appending "gpu" capability.
@@ -336,28 +357,6 @@ func ulimitsFromCompose(ulimits map[string]*types.UlimitsConfig) map[string]api.
return res
}
func devicesFromCompose(composeDevices []types.DeviceMapping) []container.DeviceMapping {
mappings := make([]container.DeviceMapping, 0, len(composeDevices))
for _, dev := range composeDevices {
mapping := container.DeviceMapping{
PathOnHost: dev.Source,
PathInContainer: dev.Target,
CgroupPermissions: dev.Permissions,
}
if mapping.PathInContainer == "" {
mapping.PathInContainer = mapping.PathOnHost
}
if mapping.CgroupPermissions == "" {
mapping.CgroupPermissions = "rwm"
}
mappings = append(mappings, mapping)
}
return mappings
}
// validateServicesExtensions validates extension combinations across all services in the project.
func validateServicesExtensions(project *types.Project) error {
for _, service := range project.Services {
pkg/client/compose/service_test.go
+43 -29
View File
@@ -1055,10 +1055,11 @@ func TestServiceSpecFromCompose_Devices(t *testing.T) {
tests := []struct {
name string
composeYAML string
expectedMappings []container.DeviceMapping
expectedDevices []api.DeviceMapping
expectedReservations []container.DeviceRequest
}{
{
name: "devices_simple",
name: "simple device",
composeYAML: `
services:
test:
@@ -1066,16 +1067,12 @@ services:
devices:
- /dev/dri
`,
expectedMappings: []container.DeviceMapping{
{
PathOnHost: "/dev/dri",
PathInContainer: "/dev/dri",
CgroupPermissions: "rwm",
},
expectedDevices: []api.DeviceMapping{
{HostPath: "/dev/dri", ContainerPath: "/dev/dri", CgroupPermissions: "rwm"},
},
},
{
name: "devices_full",
name: "device with target and permissions",
composeYAML: `
services:
test:
@@ -1083,16 +1080,12 @@ services:
devices:
- /dev/sda:/dev/xvda:r
`,
expectedMappings: []container.DeviceMapping{
{
PathOnHost: "/dev/sda",
PathInContainer: "/dev/xvda",
CgroupPermissions: "r",
},
expectedDevices: []api.DeviceMapping{
{HostPath: "/dev/sda", ContainerPath: "/dev/xvda", CgroupPermissions: "r"},
},
},
{
name: "multiple_devices",
name: "multiple devices",
composeYAML: `
services:
test:
@@ -1102,22 +1095,43 @@ services:
- /dev/sda:/dev/xvda
- "/dev/dri"
`,
expectedMappings: []container.DeviceMapping{
{
PathOnHost: "/dev/ttyUSB0",
PathInContainer: "/dev/ttyUSB0",
CgroupPermissions: "rw",
expectedDevices: []api.DeviceMapping{
{HostPath: "/dev/ttyUSB0", ContainerPath: "/dev/ttyUSB0", CgroupPermissions: "rw"},
{HostPath: "/dev/sda", ContainerPath: "/dev/xvda", CgroupPermissions: "rwm"},
{HostPath: "/dev/dri", ContainerPath: "/dev/dri", CgroupPermissions: "rwm"},
},
},
{
PathOnHost: "/dev/sda",
PathInContainer: "/dev/xvda",
CgroupPermissions: "rwm",
name: "CDI device",
composeYAML: `
services:
test:
image: nginx
devices:
- vendor.com/class=device1
`,
expectedReservations: []container.DeviceRequest{
{Driver: "cdi", DeviceIDs: []string{"vendor.com/class=device1"}},
},
},
{
PathOnHost: "/dev/dri",
PathInContainer: "/dev/dri",
CgroupPermissions: "rwm",
name: "mixed CDI and regular devices",
composeYAML: `
services:
test:
image: nginx
devices:
- /dev/dri
- vendor.com/class=device1
- nvidia.com/gpu=0
- /dev/sda:/dev/xvda:r
`,
expectedDevices: []api.DeviceMapping{
{HostPath: "/dev/dri", ContainerPath: "/dev/dri", CgroupPermissions: "rwm"},
{HostPath: "/dev/sda", ContainerPath: "/dev/xvda", CgroupPermissions: "r"},
},
expectedReservations: []container.DeviceRequest{
{Driver: "cdi", DeviceIDs: []string{"vendor.com/class=device1", "nvidia.com/gpu=0"}},
},
},
}
@@ -1130,8 +1144,8 @@ services:
spec, err := ServiceSpecFromCompose(project, "test")
require.NoError(t, err)
assert.Equal(t, tt.expectedMappings, spec.Container.Resources.DeviceMappings,
"DeviceMappings should match expected")
assert.Equal(t, tt.expectedDevices, spec.Container.Resources.Devices)
assert.Equal(t, tt.expectedReservations, spec.Container.Resources.DeviceReservations)
})
}
}
pkg/client/deploy/container.go
+1 -1
View File
@@ -87,7 +87,7 @@ func EvalContainerSpecChange(current api.ServiceSpec, new api.ServiceSpec) Conta
if !reflect.DeepEqual(current.Container.Resources.DeviceReservations, newResources.DeviceReservations) {
return ContainerNeedsRecreate
}
if !reflect.DeepEqual(current.Container.Resources.DeviceMappings, newResources.DeviceMappings) {
if !reflect.DeepEqual(current.Container.Resources.Devices, newResources.Devices) {
return ContainerNeedsRecreate
}
pkg/client/deploy/container_test.go
+6 -6
View File
@@ -1427,26 +1427,26 @@ func TestEvalContainerSpecChange_DeviceMappings(t *testing.T) {
},
{
name: "identical mapping",
current: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/foo", CgroupPermissions: "rwm"}}},
new: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/foo", CgroupPermissions: "rwm"}}},
current: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/foo", CgroupPermissions: "rwm"}}},
new: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/foo", CgroupPermissions: "rwm"}}},
want: ContainerUpToDate,
},
{
name: "add mapping",
current: api.ContainerResources{},
new: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/foo", CgroupPermissions: "rwm"}}},
new: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/foo", CgroupPermissions: "rwm"}}},
want: ContainerNeedsRecreate,
},
{
name: "remove mapping",
current: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/foo", CgroupPermissions: "rwm"}}},
current: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/foo", CgroupPermissions: "rwm"}}},
new: api.ContainerResources{},
want: ContainerNeedsRecreate,
},
{
name: "change mapping path",
current: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/foo", CgroupPermissions: "rwm"}}},
new: api.ContainerResources{DeviceMappings: []container.DeviceMapping{{PathOnHost: "/dev/foo", PathInContainer: "/dev/bar", CgroupPermissions: "rwm"}}},
current: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/foo", CgroupPermissions: "rwm"}}},
new: api.ContainerResources{Devices: []api.DeviceMapping{{HostPath: "/dev/foo", ContainerPath: "/dev/bar", CgroupPermissions: "rwm"}}},
want: ContainerNeedsRecreate,
},
}