Support vendor VFIO vGPU devices

Author: yummybomb

cmd/api/main.go

@@ -159,6 +159,32 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta
 	}, logger), nil
 }
 
+// liveInstanceVGPUDevicePaths returns the stored vGPU device paths of
+// instances whose hypervisor process is still alive. Vendor VFIO
+// reconciliation skips these VFs so it cannot clear an assignment out from
+// under a VM that survived a hypeman restart but has not opened the VFIO
+// device yet.
+func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, error) {
+	allInstances, err := instanceManager.ListInstances(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	var protected map[string]struct{}
+	for _, inst := range allInstances {
+		if inst.GPUDevicePath == "" || inst.HypervisorPID == nil {
+			continue
+		}
+		if err := syscall.Kill(*inst.HypervisorPID, 0); err != nil {
+			continue
+		}
+		if protected == nil {
+			protected = make(map[string]struct{})
+		}
+		protected[inst.GPUDevicePath] = struct{}{}
+	}
+	return protected, nil
+}
+
 func run() error {
 	// Load config early for OTel initialization
 	// Config path can be specified via CONFIG_PATH env var or defaults to platform-specific locations
@@ -349,11 +375,15 @@ func run() error {
 		return fmt.Errorf("reconcile device state: %w", err)
 	}
 
-	// Reconcile mdev devices (clears orphaned vGPUs from previous runs)
-	logger.Info("Reconciling mdev devices...")
-	if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil {
-		// Log but don't fail - mdev cleanup is best-effort
-		logger.Warn("failed to reconcile mdev devices", "error", err)
+	// Reconcile vGPU devices (clears orphaned vGPUs from previous runs)
+	logger.Info("Reconciling vGPU devices...")
+	if protected, err := liveInstanceVGPUDevicePaths(app.Ctx, app.InstanceManager); err != nil {
+		// Without the instance inventory we cannot tell live assignments from
+		// orphans, so skip rather than clear a live VM's vGPU.
+		logger.Warn("failed to list instances for vGPU reconcile protection; skipping vGPU reconciliation", "error", err)
+	} else if err := devices.ReconcileVGPUs(app.Ctx, protected); err != nil {
+		// Log but don't fail - vGPU cleanup is best-effort
+		logger.Warn("failed to reconcile vGPU devices", "error", err)
 	}
 
 	// Wire up resource validator for aggregate limit checking

integration/vgpu_test.go

@@ -245,10 +245,13 @@ func checkVGPUTestPrerequisites() (string, string) {
 		return "vGPU test requires root (sudo) for mdev creation", ""
 	}
 
-	// Check for vGPU mode (SR-IOV VFs present)
-	mode := devices.DetectHostGPUMode()
-	if mode != devices.GPUModeVGPU {
-		return "vGPU test requires SR-IOV VFs in /sys/class/mdev_bus/", ""
+	// Check for a vGPU framework (SR-IOV VFs present)
+	framework, _, err := devices.DiscoverVGPU()
+	if err != nil {
+		return "vGPU test failed to discover vGPU framework: " + err.Error(), ""
+	}
+	if framework == devices.VGPUFrameworkNone {
+		return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", ""
 	}
 
 	// Check for available profiles

lib/devices/GPU.md

@@ -8,16 +8,17 @@ hypeman supports two GPU modes, automatically detected based on host configurati
 
 | Mode | Description | Use Case |
 |------|-------------|----------|
-| **vGPU (SR-IOV)** | Virtual GPUs via mdev on SR-IOV VFs | Multi-tenant, shared GPU resources |
+| **vGPU (SR-IOV)** | Virtual GPUs on SR-IOV VFs via mdev or vendor VFIO | Multi-tenant, shared GPU resources |
 | **Passthrough** | Whole GPU VFIO passthrough | Dedicated GPU per instance |
 
 The host's GPU mode is determined by the host driver configuration:
-- If `/sys/class/mdev_bus/` contains VFs → vGPU mode
-- If NVIDIA GPUs are available for VFIO → passthrough mode
+- If `/sys/class/mdev_bus/` contains VFs → mdev vGPU mode
+- If VFs expose `/sys/bus/pci/devices/<VF>/nvidia/current_vgpu_type` → vendor VFIO vGPU mode
+- If NVIDIA GPUs are available for whole-device VFIO → passthrough mode
 
 ## vGPU Mode (Recommended)
 
-vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs), each capable of hosting an mdev (mediated device) representing a vGPU.
+vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs). Hosts on older kernels represent each vGPU as an mdev. Hosts using NVIDIA's vendor VFIO framework assign the profile directly to the VF through `current_vgpu_type`.
 
 ### How It Works
 
@@ -74,7 +75,7 @@ curl -X POST http://localhost:4973/instances \
   }'
 ```
 
-The response includes the assigned mdev UUID:
+On an mdev host, the response also includes the assigned mdev UUID:
 
 ```json
 {
@@ -87,19 +88,16 @@ The response includes the assigned mdev UUID:
 }
 ```
 
-### Ephemeral mdev Lifecycle
+### Ephemeral vGPU Lifecycle
 
-mdev devices are **ephemeral**: created on instance start, destroyed on instance delete.
+vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts and writes the profile ID/`0` to `current_vgpu_type` on vendor VFIO hosts.
 
 ```
-Instance Create → Create mdev → Attach to VM → Instance Running
-Instance Delete → Stop VM → Destroy mdev → VF available again
+Instance Create → Assign profile to VF → Attach VF to VM → Instance Running
+Instance Stop/Delete → Release profile → VF available again
 ```
 
-This ensures:
-- **Security**: No VRAM data leakage between instances
-- **Clean state**: Fresh vGPU for each instance
-- **Automatic cleanup**: Orphaned mdevs cleaned up on server restart
+Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM.
 
 ## Passthrough Mode
 
@@ -256,7 +254,8 @@ If assignment cleanup fails, Hypeman retains the instance metadata so a compatib
 
 1. Check host GPU mode detection:
    ```bash
-   ls /sys/class/mdev_bus/  # Should show VFs for vGPU mode
+   ls /sys/class/mdev_bus/
+   find /sys/bus/pci/devices -path '*/nvidia/current_vgpu_type'
    ```
 
 2. Verify NVIDIA drivers are loaded on host:
@@ -280,17 +279,18 @@ curl -s http://localhost:4973/resources | jq '.gpu.profiles'
    curl http://localhost:4973/instances/<id>/logs?source=app
    ```
 
-### mdev creation fails
+### vGPU assignment fails
 
-1. Check if VFs are available:
-   ```bash
-   ls /sys/class/mdev_bus/
-   ```
+Check the files for the framework detected on the host:
 
-2. Verify mdev types:
-   ```bash
-   cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances
-   ```
+```bash
+# mdev
+cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances
+
+# vendor VFIO
+cat /sys/bus/pci/devices/*/nvidia/creatable_vgpu_types
+cat /sys/bus/pci/devices/*/nvidia/current_vgpu_type
+```
 
 ## Performance Tuning
 

lib/devices/gpu_mode.go

@@ -15,10 +15,9 @@ func SetGPUProfileCacheTTL(ttl string) {
 	// No-op on macOS
 }
 
-// DiscoverVFs returns an empty list on macOS.
-// SR-IOV Virtual Functions are not available on macOS.
-func DiscoverVFs() ([]VirtualFunction, error) {
-	return []VirtualFunction{}, nil
+// DiscoverVGPU reports no vGPU framework on macOS.
+func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) {
+	return VGPUFrameworkNone, nil, nil
 }
 
 // ListGPUProfiles returns an empty list on macOS.
@@ -27,7 +26,7 @@ func ListGPUProfiles() ([]GPUProfile, error) {
 }
 
 // ListGPUProfilesWithVFs returns an empty list on macOS.
-func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) {
+func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) {
 	return []GPUProfile{}, nil
 }
 
@@ -62,6 +61,10 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error {
 	return nil
 }
 
+func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error {
+	return nil
+}
+
 // ReconcileMdevs is a no-op on macOS.
 func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error {
 	return nil

lib/devices/mdev_darwin.go

@@ -89,14 +89,14 @@ func getCachedProfiles(firstVF string) []profileMetadata {
 	return cachedProfiles
 }
 
-// DiscoverVFs returns all SR-IOV Virtual Functions available for vGPU.
+// discoverMdevVFs returns all SR-IOV Virtual Functions available for mdev vGPU.
 // These are discovered by scanning /sys/class/mdev_bus/ which contains
 // VFs that can host mdev devices.
-func DiscoverVFs() ([]VirtualFunction, error) {
+func discoverMdevVFs() ([]VirtualFunction, error) {
 	entries, err := os.ReadDir(mdevBusPath)
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, nil // No mdev_bus means no vGPU support
+			return nil, nil // No mdev_bus means no mdev vGPU support
 		}
 		return nil, fmt.Errorf("read mdev_bus: %w", err)
 	}
@@ -133,20 +133,9 @@ func DiscoverVFs() ([]VirtualFunction, error) {
 	return vfs, nil
 }
 
-// ListGPUProfiles returns available vGPU profiles with availability counts.
-// Profiles are discovered from the first VF's mdev_supported_types directory.
-func ListGPUProfiles() ([]GPUProfile, error) {
-	vfs, err := DiscoverVFs()
-	if err != nil {
-		return nil, err
-	}
-	return ListGPUProfilesWithVFs(vfs)
-}
-
-// ListGPUProfilesWithVFs returns available vGPU profiles using pre-discovered VFs.
-// This avoids redundant VF discovery when the caller already has the list.
-// Uses parallel sysfs reads for fast availability counting.
-func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) {
+// listMdevGPUProfilesWithVFs returns available vGPU profiles with availability
+// counts, discovered from the first VF's mdev_supported_types directory.
+func listMdevGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) {
 	if len(vfs) == 0 {
 		return nil, nil
 	}
@@ -305,7 +294,7 @@ func countAvailableForSingleProfile(freeVFsByParent map[string][]VirtualFunction
 
 // findProfileType finds the internal type name (e.g., "nvidia-556") for a profile name (e.g., "L40S-1Q")
 func findProfileType(profileName string) (string, error) {
-	vfs, err := DiscoverVFs()
+	vfs, err := discoverMdevVFs()
 	if err != nil || len(vfs) == 0 {
 		return "", fmt.Errorf("no VFs available")
 	}
@@ -531,7 +520,7 @@ func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevic
 	}
 
 	// Discover all VFs
-	vfs, err := DiscoverVFs()
+	vfs, err := discoverMdevVFs()
 	if err != nil {
 		return nil, fmt.Errorf("discover VFs: %w", err)
 	}
@@ -697,7 +686,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro
 	log := logger.FromContext(ctx)
 	_ = instanceInfos
 
-	vfs, err := DiscoverVFs()
+	vfs, err := discoverMdevVFs()
 	if err != nil {
 		return fmt.Errorf("discover managed VFs: %w", err)
 	}

lib/devices/mdev_linux.go

@@ -63,29 +63,34 @@ type GPUMode string
 type VGPUFramework string
 
 const (
-	VGPUFrameworkNone VGPUFramework = ""
-	VGPUFrameworkMdev VGPUFramework = "mdev"
+	VGPUFrameworkNone       VGPUFramework = ""
+	VGPUFrameworkMdev       VGPUFramework = "mdev"
+	VGPUFrameworkVendorVFIO VGPUFramework = "vendor-vfio"
 
 	// GPUModePassthrough indicates whole GPU VFIO passthrough
 	GPUModePassthrough GPUMode = "passthrough"
-	// GPUModeVGPU indicates SR-IOV + mdev based vGPU
+	// GPUModeVGPU indicates an mdev or vendor VFIO vGPU host
 	GPUModeVGPU GPUMode = "vgpu"
 	// GPUModeNone indicates no GPU available
 	GPUModeNone GPUMode = "none"
 )
 
 // VirtualFunction represents an SR-IOV Virtual Function for vGPU
 type VirtualFunction struct {
-	PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4"
-	ParentGPU  string `json:"parent_gpu"`  // e.g., "0000:82:00.0"
-	Allocated  bool   `json:"allocated"`   // true if a vGPU is assigned to this VF
+	PCIAddress  string `json:"pci_address"` // e.g., "0000:82:00.4"
+	ParentGPU   string `json:"parent_gpu"`  // e.g., "0000:82:00.0"
+	Allocated   bool   `json:"allocated"`   // true if a vGPU is assigned to this VF
+	ProfileType string `json:"profile_type,omitempty"`
 }
 
 // VGPUAssignment identifies an existing vGPU assignment to release.
 type VGPUAssignment struct {
 	Framework  VGPUFramework
 	DevicePath string
 	MdevUUID   string
+	// InstanceID guards vendor VFIO releases: a release is skipped when the
+	// VF's in-process owner is a different instance.
+	InstanceID string
 }
 
 type VGPUDevice struct {