test(cuda_core): report the address-space hole structure with the reservations

The largest free range alone does not predict whether both driver pools fit:
what matters is how many pool-sized reservations the free holes can hold between
them, since a reservation must fit inside one hole but a hole twice the size
takes two. Report that count on every run, and the hole addresses only when a
reservation is refused, which also shows whether the layout is being randomized.

Windows only, via VirtualQuery in a module imported only on win32; the section
is omitted elsewhere. The report is now printed once, from the terminal summary.

Author: juenglin

cuda_core/tests/conftest.py

@@ -60,7 +60,7 @@ def pytest_configure(config):
 
 
 @pytest.fixture(scope="session", autouse=True)
-def reserve_driver_pools(request, session_setup):
+def reserve_driver_pools(session_setup):
     """Take the driver's two large address-space reservations before anything else.
 
     Session-scoped and autouse so both reservations land back to back in a
@@ -81,14 +81,8 @@ def reserve_driver_pools(request, session_setup):
     with _init_cuda_context() as device:
         _reservation_report = va_reservation.reserve_driver_pools(device)
 
-    terminal_reporter = request.config.pluginmanager.get_plugin("terminalreporter")
-    if terminal_reporter is not None:
-        # Written here rather than only in the summary so the numbers survive a
-        # session that dies partway through.
-        terminal_reporter.write_sep("=", "cuda_core address space reservation")
-        for line in _reservation_report.lines():
-            terminal_reporter.write_line(line)
-
+    # Reported once, from pytest_terminal_summary. On the abort path below the
+    # message carries the same numbers, so nothing is lost by not printing here.
     if _reservation_report.failed:
         pytest.exit(va_reservation.build_failure_message(_reservation_report), returncode=1)
 

cuda_core/tests/helpers/va_reservation.py

@@ -30,10 +30,21 @@
 from __future__ import annotations
 
 import os
+import sys
 import time
 
 from cuda.bindings import driver
 
+if sys.platform == "win32":
+    from helpers import win_address_space
+else:
+    # No Linux counterpart on purpose; see helpers/win_address_space.py.
+    win_address_space = None
+
+# Holes smaller than this are noise in the layout dump.
+LAYOUT_MIN_HOLE = 1024 * 1024 * 1024
+LAYOUT_MAX_HOLES = 6
+
 MIB = 1024 * 1024
 GIB = 1024 * MIB
 
@@ -130,6 +141,44 @@ def format_bytes(value: int | None) -> str:
     return f"{value / GIB:.2f} GiB"
 
 
+def free_holes() -> list[tuple[int, int]] | None:
+    """Unallocated holes as ``(size, base)``, largest first. None off Windows."""
+    if win_address_space is None:
+        return None
+    return win_address_space.free_regions(LAYOUT_MIN_HOLE)
+
+
+def pool_capacity(holes, pool_bytes) -> int | None:
+    """How many driver pools the free holes could hold between them.
+
+    This is the number that decides the outcome, and it is neither the largest
+    hole nor the count of holes that clear one pool. A reservation has to fit
+    within a single hole, but a hole twice the size takes two -- the driver
+    packs them from its low end, so a lone 800 GiB hole hosts both pools just
+    as well as two 400 GiB ones. Summing each hole's capacity covers both.
+    """
+    if holes is None or not pool_bytes:
+        return None
+    return sum(size // pool_bytes for size, _base in holes)
+
+
+def layout_lines(holes, pool_bytes, label, detail: bool = True) -> list[str]:
+    """Render the hole structure. Base addresses show whether it is randomized."""
+    if holes is None:
+        return []
+    capacity = pool_capacity(holes, pool_bytes)
+    headline = f"free holes {label}: {len(holes)} >= {format_bytes(LAYOUT_MIN_HOLE)}"
+    if capacity is not None:
+        headline += f", room for {capacity} pool(s) of {format_bytes(pool_bytes)}"
+    out = [headline]
+    if detail:
+        for size, base in holes[:LAYOUT_MAX_HOLES]:
+            out.append(f"  {format_bytes(size):>14}  @ {base:#018x}")
+        if len(holes) > LAYOUT_MAX_HOLES:
+            out.append(f"  ... and {len(holes) - LAYOUT_MAX_HOLES} smaller")
+    return out
+
+
 class Reservation:
     """One driver-managed pool that has to be materialized."""
 
@@ -211,7 +260,16 @@ class ReservationReport:
     """What the early reservations cost, for the terminal."""
 
     def __init__(
-        self, device_name, device_memory, before, after, reservations, measured, seconds=0.0, unsupported=False
+        self,
+        device_name,
+        device_memory,
+        before,
+        after,
+        reservations,
+        measured,
+        seconds=0.0,
+        unsupported=False,
+        holes_before=None,
     ):
         self.device_name = device_name
         self.device_memory = device_memory
@@ -221,6 +279,7 @@ def __init__(
         self.measured = measured
         self.seconds = seconds
         self.unsupported = unsupported
+        self.holes_before = holes_before
 
     @property
     def failed(self) -> list[Reservation]:
@@ -264,6 +323,11 @@ def lines(self) -> list[str]:
                     f"remaining headroom: {self.after // pool_bytes} more pool-sized "
                     f"({format_bytes(pool_bytes)}) reservations  [{self.seconds:.1f}s measuring]"
                 )
+        # Just the counts here. They are what makes a successful session
+        # comparable with a failed one, since the hole count is what decides the
+        # outcome. The addresses behind them are only worth printing when a
+        # reservation is actually refused; see build_failure_message.
+        out += layout_lines(self.holes_before, self.pool_reservation_bytes, "at session start", detail=False)
         return out
 
 
@@ -285,6 +349,14 @@ def build_failure_message(report: ReservationReport) -> str:
     for item in report.failed:
         lines.append(f"  {item.name} ({item.detail}): {item.error}")
 
+    # Each pool needs a hole of its own, so the hole structure -- not the total
+    # free -- is what decides this. Included here because it is the first thing
+    # anyone diagnosing a refusal will want.
+    layout = layout_lines(report.holes_before, pool_bytes, "at session start")
+    if layout:
+        lines.append("")
+        lines += [f"  {line}" for line in layout]
+
     return "\n".join(lines)
 
 
@@ -307,6 +379,7 @@ def reserve_driver_pools(device, measure: bool = True) -> ReservationReport:
     started = time.perf_counter()
     before = largest_reservable() if measured else None
     elapsed = time.perf_counter() - started
+    holes_before = free_holes()
 
     reservations = reservations_for(device)
     for item in reservations:
@@ -315,4 +388,13 @@ def reserve_driver_pools(device, measure: bool = True) -> ReservationReport:
     started = time.perf_counter()
     after = largest_reservable() if measured else None
     elapsed += time.perf_counter() - started
-    return ReservationReport(device.name, device_memory, before, after, reservations, measured, elapsed)
+    return ReservationReport(
+        device.name,
+        device_memory,
+        before,
+        after,
+        reservations,
+        measured,
+        elapsed,
+        holes_before=holes_before,
+    )

cuda_core/tests/helpers/win_address_space.py

@@ -0,0 +1,74 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Read this process's virtual address space layout on Windows.
+
+Only imported on Windows -- see helpers/va_reservation.py, which guards the
+import. There is deliberately no Linux counterpart: the address-space pressure
+this exists to diagnose (issue #2381) is specific to the bounded per-process
+budget on Windows, and on Linux the budget is large enough that the layout is
+not interesting.
+
+``cuMemAddressReserve`` probing can only report the largest single hole. That
+turned out to be the wrong number: two pool reservations do not need one hole
+twice their size, they need *two* holes, so a session can start with a smaller
+largest-hole and still succeed. Walking the address space shows the whole hole
+structure, which is what actually predicts the outcome.
+"""
+
+from __future__ import annotations
+
+import ctypes
+from ctypes import wintypes
+
+MEM_COMMIT = 0x1000
+MEM_RESERVE = 0x2000
+MEM_FREE = 0x10000
+
+# User-mode address space ceiling; walking past it wastes time and returns nothing.
+_USER_SPACE_LIMIT = 1 << 47
+
+
+class MEMORY_BASIC_INFORMATION(ctypes.Structure):
+    _fields_ = [
+        ("BaseAddress", ctypes.c_void_p),
+        ("AllocationBase", ctypes.c_void_p),
+        ("AllocationProtect", wintypes.DWORD),
+        ("PartitionId", wintypes.WORD),
+        ("__alignment", wintypes.WORD),
+        ("RegionSize", ctypes.c_size_t),
+        ("State", wintypes.DWORD),
+        ("Protect", wintypes.DWORD),
+        ("Type", wintypes.DWORD),
+    ]
+
+
+_kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+_kernel32.VirtualQuery.argtypes = [ctypes.c_void_p, ctypes.POINTER(MEMORY_BASIC_INFORMATION), ctypes.c_size_t]
+_kernel32.VirtualQuery.restype = ctypes.c_size_t
+
+
+def walk():
+    """Yield ``(base, size, state)`` for every region in this process."""
+    info = MEMORY_BASIC_INFORMATION()
+    address = 0
+    while address < _USER_SPACE_LIMIT:
+        if not _kernel32.VirtualQuery(ctypes.c_void_p(address), ctypes.byref(info), ctypes.sizeof(info)):
+            break
+        size = info.RegionSize
+        if size == 0:
+            break
+        yield address, size, info.State
+        address += size
+
+
+def free_regions(min_size: int = 0) -> list[tuple[int, int]]:
+    """``(size, base)`` for every unallocated hole, largest first."""
+    holes = [(size, base) for base, size, state in walk() if state == MEM_FREE and size >= min_size]
+    holes.sort(reverse=True)
+    return holes
+
+
+def reserved_total() -> int:
+    """Bytes reserved but not committed, i.e. address space held without memory."""
+    return sum(size for _base, size, state in walk() if state == MEM_RESERVE)

cuda_core/tests/test_helpers.py

@@ -160,9 +160,78 @@ def test_failure_message_names_the_pool_and_the_size_it_needed():
 
     assert "cannot run on this machine" in message
     assert "graph memory pool" in message
+    assert "cuGraphAddMemAllocNode" in message
     assert "357.63 GiB" in message  # 2x installed device memory
     assert "address space" in message
-    assert "#2381" in message
+    # The driver's own error, so the reader is not left guessing what refused.
+    assert "ZeroDivisionError" in message
+
+
+@pytest.mark.agent_authored(model="claude-opus-5")
+def test_pool_capacity_counts_two_pools_in_one_large_hole():
+    # A reservation must fit within a single hole, but one hole twice the size
+    # takes both pools -- the driver packs them from its low end. Counting only
+    # the holes that clear one pool would call this a failure.
+    pool = 358 * va_reservation.GIB
+    one_big_hole = [(800 * va_reservation.GIB, 0x1000)]
+    two_holes = [(400 * va_reservation.GIB, 0x1000), (400 * va_reservation.GIB, 0x2000)]
+
+    assert va_reservation.pool_capacity(one_big_hole, pool) == 2
+    assert va_reservation.pool_capacity(two_holes, pool) == 2
+
+
+@pytest.mark.agent_authored(model="claude-opus-5")
+def test_pool_capacity_ignores_holes_that_cannot_take_a_whole_pool():
+    # Free space that is merely plentiful does not help; it has to be
+    # contiguous. This is the shape that fails on the affected machine.
+    pool = 358 * va_reservation.GIB
+    lopsided = [(700 * va_reservation.GIB, 0x1000), (300 * va_reservation.GIB, 0x2000)]
+
+    assert va_reservation.pool_capacity(lopsided, pool) == 1
+
+
+@pytest.mark.agent_authored(model="claude-opus-5")
+def test_layout_lines_are_empty_off_windows():
+    # free_holes() returns None where VirtualQuery is unavailable; the report
+    # must simply omit the section rather than fail.
+    assert va_reservation.layout_lines(None, 358 * va_reservation.GIB, "before") == []
+    assert va_reservation.pool_capacity(None, 358 * va_reservation.GIB) is None
+
+
+@pytest.mark.agent_authored(model="claude-opus-5")
+def test_neither_report_mentions_holes_off_windows(monkeypatch):
+    # The hole layout comes from VirtualQuery, so it is Windows-only. Everything
+    # else is CUDA APIs and must still be reported. Simulates the non-win32
+    # branch of the guarded import in va_reservation.
+    monkeypatch.setattr(va_reservation, "win_address_space", None)
+    good = va_reservation.Reservation("default device mempool", "cuDeviceGetMemPool", lambda: None)
+    bad = va_reservation.Reservation("graph memory pool", "cuGraphAddMemAllocNode", lambda: 1 / 0)
+    good.run()
+    bad.run()
+    holes = va_reservation.free_holes()
+    gib = va_reservation.GIB
+
+    report = va_reservation.ReservationReport(
+        "dev", 25650855936, 900 * gib, 800 * gib, [good, bad], True, holes_before=holes
+    )
+
+    assert holes is None
+    assert "free holes" not in "\n".join(report.lines())
+    message = va_reservation.build_failure_message(report)
+    assert "free holes" not in message
+    assert "graph memory pool" in message  # the rest of the report survives
+
+
+@pytest.mark.agent_authored(model="claude-opus-5")
+def test_layout_lines_report_base_addresses():
+    # The bases are the point: comparing them across launches shows whether the
+    # layout is being randomized.
+    holes = [(500 * va_reservation.GIB, 0x2200000000)]
+
+    text = "\n".join(va_reservation.layout_lines(holes, 358 * va_reservation.GIB, "before"))
+
+    assert "room for 1 pool(s)" in text
+    assert "0x0000002200000000" in text
 
 
 @pytest.mark.agent_authored(model="claude-opus-5")