Skip to content

Commit ee1b160

Browse files
authored
Convict on a released assignment instead of skipping
The pre-conviction recheck treated a missing assignment like a changed one and skipped the quarantine. A stop or delete landing between the metadata snapshot and the recheck releases the assignment but cannot un-wedge the VF, and the archive-before-persist ordering already ties a marker read under this epoch's snapshot to this epoch's VF, so the conviction is valid. Skip only when a different assignment exists.
1 parent ef934fb commit ee1b160

2 files changed

Lines changed: 11 additions & 8 deletions

File tree

lib/instances/vgpu_sentinel.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -231,19 +231,20 @@ func (c *VGPUSentinelController) convict(ctx context.Context, target vgpuSentine
231231
// The target came from a metadata snapshot taken before the log read: a
232232
// stop/start racing the scan could have assigned a different VF by now,
233233
// and convicting the snapshot's VF would quarantine a healthy device.
234-
// Reload the assignment and convict only if it is unchanged. Start
234+
// Reload the assignment and skip only if a different one exists. Start
235235
// archives the previous boot's log before persisting a new assignment,
236-
// so if the metadata still shows this epoch after the marker was read,
237-
// the marker provably belongs to this epoch's VF. On any other outcome
238-
// the tail stays open; the next scan sees the current assignment and
239-
// rescans its log from the top.
236+
// so a marker read under this epoch's snapshot provably belongs to this
237+
// epoch's VF — including when the instance was stopped or deleted after
238+
// the read, which releases the assignment but cannot un-wedge the VF.
239+
// On a skip or error the tail stays open; the next scan sees the current
240+
// assignment and rescans its log from the top.
240241
current, ok, err := c.store.getVGPUSentinelTarget(ctx, target.instanceID)
241242
if err != nil {
242243
c.log.WarnContext(ctx, "vGPU sentinel could not confirm assignment before conviction",
243244
"vf", target.vfAddress, "instance_id", target.instanceID, "error", err)
244245
return false
245246
}
246-
if !ok || current.vfAddress != target.vfAddress || current.assignedAt != target.assignedAt {
247+
if ok && (current.vfAddress != target.vfAddress || current.assignedAt != target.assignedAt) {
247248
c.log.InfoContext(ctx, "vGPU sentinel skipping conviction: assignment changed during scan",
248249
"vf", target.vfAddress, "instance_id", target.instanceID)
249250
return false

lib/instances/vgpu_sentinel_test.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -268,11 +268,13 @@ func TestVGPUSentinelControllerSkipsConvictionOnChangedAssignment(t *testing.T)
268268
assert.Empty(t, *quarantined)
269269
assert.False(t, c.tails["instance-1"].done)
270270

271-
// A deleted instance can no longer vouch for the marker either.
271+
// A stop or delete after the marker was read releases the assignment but
272+
// cannot un-wedge the VF: the marker still convicts the scanned epoch's VF.
272273
store.targets = nil
273274
c.tails["instance-1"] = &vgpuSentinelTail{vfAddress: stale.vfAddress, assignedAt: stale.assignedAt}
274275
c.scanTarget(context.Background(), stale)
275-
assert.Empty(t, *quarantined)
276+
require.Len(t, *quarantined, 1)
277+
assert.Equal(t, "0000:e3:00.4", (*quarantined)[0].VFAddress)
276278
}
277279

278280
func TestListVGPUSentinelTargetsSkipsUnstattableMetadata(t *testing.T) {

0 commit comments

Comments
 (0)