Skip to content

Commit e61e5a3

Browse files
committed
Verify socket ownership before treating a vGPU claimant PID as live
A bare kill(0) check treats any process that reused a stored hypervisor PID as the owning VMM, so a reused PID could keep an orphaned VF in the startup protected set or make a stale record look like a live claim during release. Require the PID to own the instance's hypervisor socket on Linux in both paths. Also retain assignment metadata when start's vGPU rollback fails, matching create and stop, so later release paths can still find the device path instead of leaking the slot until the next restart.
1 parent 19b9602 commit e61e5a3

6 files changed

Lines changed: 45 additions & 3 deletions

File tree

cmd/api/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.
174174
if inst.GPUDevicePath == "" || inst.HypervisorPID == nil {
175175
continue
176176
}
177-
if !instances.ProcessExists(*inst.HypervisorPID) {
177+
if !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
178178
continue
179179
}
180180
protected[inst.GPUDevicePath] = struct{}{}

lib/instances/lifecycle_noop_test.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package instances
33
import (
44
"context"
55
"errors"
6+
"net"
67
"os"
78
"path/filepath"
89
"sync"
@@ -183,14 +184,18 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) {
183184
claimantID := "inst-live-claimant"
184185
require.NoError(t, m.ensureDirectories(claimantID))
185186
pid := os.Getpid()
187+
socketPath := m.paths.InstanceSocket(claimantID, "noop.sock")
188+
listener, err := net.Listen("unix", socketPath)
189+
require.NoError(t, err)
190+
defer listener.Close()
186191
require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
187192
Id: claimantID,
188193
Name: claimantID,
189194
Image: "test-image",
190195
CreatedAt: now,
191196
HypervisorType: lifecycleNoopHypervisorType,
192197
HypervisorPID: &pid,
193-
SocketPath: m.paths.InstanceSocket(claimantID, "noop.sock"),
198+
SocketPath: socketPath,
194199
DataDir: m.paths.InstanceDir(claimantID),
195200
GPUProfile: "NVIDIA L40S-2Q",
196201
GPUFramework: devices.VGPUFramework("future-framework"),
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
//go:build linux
2+
3+
package instances
4+
5+
import (
6+
"os"
7+
"path/filepath"
8+
"testing"
9+
10+
"github.com/stretchr/testify/assert"
11+
)
12+
13+
func TestHypervisorProcessExistsRejectsLivePIDWithoutSocketOwnership(t *testing.T) {
14+
t.Parallel()
15+
16+
assert.False(t, HypervisorProcessExists(os.Getpid(), filepath.Join(t.TempDir(), "missing.sock")))
17+
}

lib/instances/query.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,23 @@ func refreshHypervisorPID(stored *StoredMetadata, state State) {
587587
}
588588
}
589589

590+
// HypervisorProcessExists reports whether pid owns the instance's hypervisor
591+
// socket. On Linux this distinguishes a surviving hypervisor from an unrelated
592+
// process that reused its PID.
593+
func HypervisorProcessExists(pid int, socketPath string) bool {
594+
if !ProcessExists(pid) {
595+
return false
596+
}
597+
if runtime.GOOS != "linux" {
598+
return true
599+
}
600+
if socketPath == "" {
601+
return false
602+
}
603+
resolvedPID, err := hypervisor.ResolveProcessPID(socketPath)
604+
return err == nil && resolvedPID == pid
605+
}
606+
590607
// ProcessExists reports whether pid belongs to a live, non-zombie process.
591608
func ProcessExists(pid int) bool {
592609
if pid <= 0 {

lib/instances/start.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,9 @@ func (m *manager) startInstance(
178178
}
179179
if err := devices.DestroyVGPU(ctx, assignment); err != nil {
180180
log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err)
181+
if saveErr := m.saveMetadata(meta); saveErr != nil {
182+
log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr)
183+
}
181184
}
182185
})
183186
if err := m.saveMetadata(meta); err != nil {

lib/instances/vgpu.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu
6767
if inst.Id == excludeID || inst.GPUDevicePath != devicePath || inst.HypervisorPID == nil {
6868
continue
6969
}
70-
if ProcessExists(*inst.HypervisorPID) {
70+
if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
7171
return true, nil
7272
}
7373
}

0 commit comments

Comments
 (0)