Keep vGPU placement available when an allocated type is unknown

Sort GPUs with unaccountable load last instead of rejecting placement, and stop reporting passthrough capacity when vGPU discovery fails.

Author: yummybomb

lib/devices/GPU.md

@@ -239,9 +239,10 @@ Before downgrading Hypeman or the host to a version that does not support the ac
 
 1. Stop or delete all vGPU instances while the current Hypeman version can release their assignments.
 2. Confirm `/resources` reports `used_slots: 0`.
-3. Confirm no mdev assignments remain:
+3. Confirm no assignments remain in either framework:
    ```bash
    test -z "$(find /sys/bus/mdev/devices -mindepth 1 -maxdepth 1 2>/dev/null)"
+   find /sys/bus/pci/devices -path '*/nvidia/current_vgpu_type' -exec grep -H -v '^0$' {} +
    ```
 4. Downgrade only after both checks are clean.
 

lib/devices/vendor_vfio_linux.go

@@ -279,12 +279,19 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map
 
 func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) {
 	usageByGPU := make(map[string]int)
+	unknownUsageByGPU := make(map[string]bool)
 	freeByGPU := make(map[string][]VirtualFunction)
 	for _, vf := range vfs {
 		if vf.Allocated {
+			// framebufferByType only covers currently creatable profiles, so
+			// after a restart an allocated type can be missing when its
+			// capacity is exhausted. Prefer GPUs whose load is fully known
+			// instead of rejecting placement outright; the kernel driver
+			// still enforces real capacity through creatable_vgpu_types.
 			framebuffer, ok := s.framebufferByType[vf.ProfileType]
 			if !ok {
-				return "", fmt.Errorf("framebuffer size for allocated vGPU type %s is unknown", vf.ProfileType)
+				unknownUsageByGPU[vf.ParentGPU] = true
+				continue
 			}
 			usageByGPU[vf.ParentGPU] += framebuffer
 			continue
@@ -306,6 +313,9 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType
 		gpus = append(gpus, gpu)
 	}
 	sort.Slice(gpus, func(i, j int) bool {
+		if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] {
+			return !unknownUsageByGPU[gpus[i]]
+		}
 		if usageByGPU[gpus[i]] == usageByGPU[gpus[j]] {
 			return gpus[i] < gpus[j]
 		}

lib/devices/vendor_vfio_linux_test.go

@@ -252,6 +252,33 @@ func TestVendorVFIOSelectsLeastLoadedGPUWithConsumedType(t *testing.T) {
 	assert.Equal(t, "0000:e3:00.4", device.VFAddress)
 }
 
+func TestVendorVFIOPlacementPrefersKnownLoadWhenAllocatedTypeIsUnknown(t *testing.T) {
+	t.Parallel()
+
+	// Simulates a restart: type 1159 is allocated but no longer creatable
+	// anywhere, so its framebuffer size is unknown.
+	sysfs := newTestVendorVFIOSysfs(t)
+	sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "")
+	sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147  : NVIDIA L40S-1Q\n")
+	sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", "1147  : NVIDIA L40S-1Q\n")
+
+	device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1")
+	require.NoError(t, err)
+	assert.Equal(t, "0000:e3:00.4", device.VFAddress, "the GPU with unknown load should be picked last")
+}
+
+func TestVendorVFIOPlacesOnGPUWithUnknownLoadWhenItHasTheOnlyCapacity(t *testing.T) {
+	t.Parallel()
+
+	sysfs := newTestVendorVFIOSysfs(t)
+	sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "")
+	sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147  : NVIDIA L40S-1Q\n")
+
+	device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1")
+	require.NoError(t, err)
+	assert.Equal(t, "0000:82:00.5", device.VFAddress)
+}
+
 func TestVendorVFIOReconcile(t *testing.T) {
 	t.Parallel()
 

lib/resources/gpu.go

@@ -22,9 +22,12 @@ type GPUResourceStatus struct {
 func GetGPUStatus(ctx context.Context) *GPUResourceStatus {
 	framework, vfs, err := devices.DiscoverVGPU()
 	if err != nil {
-		// A failed vGPU probe must not hide passthrough GPUs from status reporting.
+		// Only report passthrough once vGPU discovery confirms no vGPU
+		// framework. On a vGPU host a transient probe failure would otherwise
+		// expose the PFs/VFs as available passthrough slots while active vGPU
+		// assignments exist.
 		logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err)
-		return getPassthroughStatus()
+		return nil
 	}
 	if framework != devices.VGPUFrameworkNone {
 		return getVGPUStatus(ctx, framework, vfs)