Report vendor VFIO availability per parent GPU and thread instance ownership through releases

Author: yummybomb

lib/devices/vendor_vfio_linux.go

@@ -81,9 +81,13 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) {
 	return vfs, nil
 }
 
+// listProfiles aggregates creatable profiles per parent GPU. Free VFs on the
+// same GPU share its framebuffer, so counting each advertising VF overreports
+// availability. The driver only guarantees that a GPU still advertising a
+// type can fit one more instance of it, so report that per-GPU lower bound.
 func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) {
 	profilesByType := make(map[string]profileMetadata)
-	availability := make(map[string]int)
+	creatableGPUs := make(map[string]map[string]struct{})
 	for _, vf := range vfs {
 		creatable, err := s.readCreatableProfiles(vf.PCIAddress)
 		if err != nil {
@@ -92,7 +96,14 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro
 		for _, profile := range creatable {
 			profilesByType[profile.TypeName] = profile
 			if !vf.Allocated {
-				availability[profile.TypeName]++
+				gpu := vf.ParentGPU
+				if gpu == "" {
+					gpu = vf.PCIAddress
+				}
+				if creatableGPUs[profile.TypeName] == nil {
+					creatableGPUs[profile.TypeName] = make(map[string]struct{})
+				}
+				creatableGPUs[profile.TypeName][gpu] = struct{}{}
 			}
 		}
 	}
@@ -108,7 +119,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro
 		profiles = append(profiles, GPUProfile{
 			Name:          profile.Name,
 			FramebufferMB: profile.FramebufferMB,
-			Available:     availability[profile.TypeName],
+			Available:     len(creatableGPUs[profile.TypeName]),
 		})
 	}
 	return profiles, nil

lib/devices/vendor_vfio_linux_test.go

@@ -184,6 +184,22 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T
 	}
 }
 
+func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) {
+	t.Parallel()
+
+	sysfs := newTestVendorVFIOSysfs(t)
+	sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes)
+	sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes)
+	sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes)
+
+	vfs, err := sysfs.discoverVFs()
+	require.NoError(t, err)
+	profiles, err := sysfs.listProfiles(vfs)
+	require.NoError(t, err)
+	assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"),
+		"free VFs share their parent GPU's capacity, so availability is per GPU")
+}
+
 func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) {
 	t.Parallel()
 

lib/instances/create.go

@@ -302,6 +302,7 @@ func (m *manager) createInstance(
 				Framework:  gpuDevice.Framework,
 				DevicePath: gpuDevice.SysfsPath,
 				MdevUUID:   gpuDevice.MdevUUID,
+				InstanceID: id,
 			}
 			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
 				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err)

lib/instances/start.go

@@ -174,6 +174,7 @@ func (m *manager) startInstance(
 				Framework:  device.Framework,
 				DevicePath: device.SysfsPath,
 				MdevUUID:   device.MdevUUID,
+				InstanceID: id,
 			}
 			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
 				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err)

lib/instances/vgpu.go

@@ -27,6 +27,7 @@ func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error {
 			Framework:  stored.GPUFramework,
 			DevicePath: path,
 			MdevUUID:   stored.GPUMdevUUID,
+			InstanceID: stored.Id,
 		}
 		if err := devices.DestroyVGPU(ctx, assignment); err != nil {
 			return err