Add check and agent guidance about uncapped mempools

cuda_core/tests/AGENTS.md

@@ -0,0 +1,67 @@
+# cuda.core test suite
+
+Package-wide conventions live in `../AGENTS.md`; repository-wide ones in
+`../../AGENTS.md`. This file covers conventions specific to the tests.
+
+## Never create an uncapped memory pool
+
+A memory pool created without `max_size` reserves a virtual address window
+sized from device memory (roughly 1x device memory) regardless of what the
+test actually allocates. The reservation is charged to the process address
+space even though it is not backed by physical memory, and it is not returned
+until the pool is destroyed *and* the stream-ordered frees of its outstanding
+allocations retire. The whole suite shares one process and one device, so
+these reservations accumulate across tests.
+
+When a test needs its own pool, use the suite-wide cap:
+
+```python
+POOL_SIZE = 2097152  # 2 MiB
+
+mr = DeviceMemoryResource(dev, DeviceMemoryResourceOptions(max_size=POOL_SIZE))
+```
+
+Use a larger value only if a test genuinely requires it.
+
+### Passing no options is different from passing empty options
+
+`DeviceMemoryResource(dev)` with no options does **not** create a pool. It
+wraps the device's existing default mempool (`_mempool_owned` is false) and
+costs no additional address space. Passing *any* options object creates a new
+owned pool, and a new pool without `max_size` is uncapped:
+
+```python
+DeviceMemoryResource(dev)                                     # wraps default pool, free
+DeviceMemoryResource(dev, DeviceMemoryResourceOptions())      # NEW uncapped pool, expensive
+DeviceMemoryResource(dev, {"ipc_enabled": True})              # NEW uncapped pool, expensive
+```
+
+Do not add `max_size` to a call that currently passes no options: that
+converts a free default-pool wrapper into a new pool and makes things worse.
+
+### Managed pools are exempt
+
+`cuMemPoolCreate` requires `CUmemPoolProps.maxSize` to be zero for managed
+pools, so `ManagedMemoryResourceOptions` has no `max_size` option. Managed
+pools cannot be right-sized and are not checked.
+
+### Enforcement
+
+`test_mempool_hygiene.py` statically scans this directory and fails on
+`DeviceMemoryResourceOptions` / `PinnedMemoryResourceOptions` constructions
+that omit `max_size`. When a call is deliberately exempt -- most often because
+it sits inside `pytest.raises` and no pool is ever created -- annotate it:
+
+```python
+with pytest.raises(RuntimeError, match="IPC is not available"):
+    # uncapped-pool-ok: raises before the pool is created
+    DeviceMemoryResource(mempool_device, DeviceMemoryResourceOptions(ipc_enabled=True))
+```
+
+## Release resources at test boundaries
+
+The `_init_cuda_context` fixture in `conftest.py` runs `gc.collect()` followed
+by `cuCtxSynchronize()` before popping the context. Tests should not rely on
+that as a substitute for cleaning up explicitly: prefer context managers for
+resources whose lifetime fits a single scope, and keep pool lifetimes inside
+the test that creates them.

cuda_core/tests/test_memory.py

@@ -1519,9 +1519,11 @@ def test_pinned_mr_numa_id_negative_error(init_cuda):
     skip_if_pinned_memory_unsupported(device)
 
     with pytest.raises(ValueError, match="numa_id must be >= 0"):
+        # uncapped-pool-ok: numa_id is validated before the pool is created
         PinnedMemoryResource(PinnedMemoryResourceOptions(numa_id=-1))
 
     with pytest.raises(ValueError, match="numa_id must be >= 0"):
+        # uncapped-pool-ok: numa_id is validated before the pool is created
         PinnedMemoryResource(PinnedMemoryResourceOptions(numa_id=-42))
 
 
@@ -1971,4 +1973,5 @@ def test_dmr_ipc_enabled_unsupported_raises(mempool_device):
     if not IS_WINDOWS:
         pytest.skip("memory IPC is supported on this platform; unsupported-raise path is Windows-only")
     with pytest.raises(RuntimeError, match="IPC is not available"):
+        # uncapped-pool-ok: IPC support is checked before the pool is created
         DeviceMemoryResource(mempool_device, DeviceMemoryResourceOptions(ipc_enabled=True))

cuda_core/tests/test_mempool_hygiene.py

@@ -0,0 +1,90 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Static guard against uncapped memory pools in the test suite.
+
+A pool created without ``max_size`` reserves an address-space window sized from
+device memory rather than from what the test allocates, and the whole suite
+shares one process. Enough of those reservations exhaust the address space and
+the rest of the session fails with ``CUDA_ERROR_OUT_OF_MEMORY`` on a device with
+free physical memory (issue #2381). See AGENTS.md in this directory.
+
+This check is static rather than runtime so that it also covers pools created
+by tests that are skipped on the current platform.
+"""
+
+import ast
+import pathlib
+
+import pytest
+
+TESTS_ROOT = pathlib.Path(__file__).parent
+
+# Managed pools cannot be right-sized: cuMemPoolCreate requires maxSize == 0 for
+# managed pools, so ManagedMemoryResourceOptions has no max_size to set.
+CAPPABLE_OPTIONS = frozenset({"DeviceMemoryResourceOptions", "PinnedMemoryResourceOptions"})
+CAPPABLE_RESOURCES = frozenset({"DeviceMemoryResource", "PinnedMemoryResource"})
+
+OPT_OUT_MARKER = "uncapped-pool-ok"
+
+
+def _callee_name(node: ast.Call) -> str:
+    func = node.func
+    if isinstance(func, ast.Attribute):
+        return func.attr
+    if isinstance(func, ast.Name):
+        return func.id
+    return ""
+
+
+def _is_capped(node: ast.Call) -> bool:
+    # ``**kwargs`` (arg is None) may carry max_size; do not guess.
+    return any(kw.arg is None or kw.arg == "max_size" for kw in node.keywords)
+
+
+def _dict_is_capped(node: ast.Dict) -> bool:
+    for key in node.keys:
+        if key is None:  # ``**other`` inside the literal
+            return True
+        if isinstance(key, ast.Constant) and key.value == "max_size":
+            return True
+    return False
+
+
+def _opted_out(lines: list[str], node: ast.AST) -> bool:
+    """True if the call, or the line above it, carries the opt-out marker."""
+    start = max(node.lineno - 2, 0)  # -1 for 0-based, -1 more for a preceding comment
+    end = getattr(node, "end_lineno", node.lineno)
+    return any(OPT_OUT_MARKER in line for line in lines[start:end])
+
+
+def _violations_in(path: pathlib.Path) -> list[str]:
+    source = path.read_text(encoding="utf-8")
+    lines = source.splitlines()
+    found = []
+    for node in ast.walk(ast.parse(source, filename=str(path))):
+        if not isinstance(node, ast.Call):
+            continue
+        name = _callee_name(node)
+        if name in CAPPABLE_OPTIONS:
+            uncapped = not _is_capped(node)
+        elif name in CAPPABLE_RESOURCES:
+            # The options may also be given as a dict literal.
+            dicts = [arg for arg in [*node.args, *(kw.value for kw in node.keywords)] if isinstance(arg, ast.Dict)]
+            uncapped = any(not _dict_is_capped(d) for d in dicts)
+        else:
+            continue
+        if uncapped and not _opted_out(lines, node):
+            found.append(f"{path.relative_to(TESTS_ROOT).as_posix()}:{node.lineno}: {name} without max_size")
+    return found
+
+
+@pytest.mark.agent_authored(model="claude-opus-5")
+def test_no_uncapped_memory_pools():
+    violations = sorted(v for path in TESTS_ROOT.rglob("*.py") for v in _violations_in(path))
+    assert not violations, (
+        "Memory pools created by tests must set max_size (use POOL_SIZE = 2097152).\n"
+        f"Annotate a deliberate exception with a '# {OPT_OUT_MARKER}: <reason>' comment.\n"
+        "See cuda_core/tests/AGENTS.md.\n" + "\n".join(violations)
+    )

cuda_core/tests/test_multiprocessing_warning.py

@@ -18,12 +18,14 @@
 from cuda.core._memory._ipc import _reduce_allocation_handle
 from cuda.core._utils.cuda_utils import check_multiprocessing_start_method, reset_fork_warning
 
+POOL_SIZE = 2097152  # 2MB size
+
 
 def test_warn_on_fork_method_device_memory_resource(ipc_device):
     """Test that warning is emitted when DeviceMemoryResource is pickled with fork method."""
     device = ipc_device
     device.set_current()
-    options = DeviceMemoryResourceOptions(max_size=2097152, ipc_enabled=True)
+    options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True)
     mr = DeviceMemoryResource(device, options=options)
 
     with patch("multiprocessing.get_start_method", return_value="fork"), warnings.catch_warnings(record=True) as w:
@@ -50,7 +52,7 @@ def test_warn_on_fork_method_allocation_handle(ipc_device):
     """Test that warning is emitted when IPCAllocationHandle is pickled with fork method."""
     device = ipc_device
     device.set_current()
-    options = DeviceMemoryResourceOptions(max_size=2097152, ipc_enabled=True)
+    options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True)
     mr = DeviceMemoryResource(device, options=options)
     alloc_handle = mr.allocation_handle
 
@@ -102,7 +104,7 @@ def test_no_warning_with_spawn_method(ipc_device):
     """Test that no warning is emitted when start method is 'spawn'."""
     device = ipc_device
     device.set_current()
-    options = DeviceMemoryResourceOptions(max_size=2097152, ipc_enabled=True)
+    options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True)
     mr = DeviceMemoryResource(device, options=options)
 
     with patch("multiprocessing.get_start_method", return_value="spawn"), warnings.catch_warnings(record=True) as w:
@@ -125,7 +127,7 @@ def test_warning_emitted_only_once(ipc_device):
     """Test that warning is only emitted once even when multiple objects are pickled."""
     device = ipc_device
     device.set_current()
-    options = DeviceMemoryResourceOptions(max_size=2097152, ipc_enabled=True)
+    options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True)
     mr1 = DeviceMemoryResource(device, options=options)
     mr2 = DeviceMemoryResource(device, options=options)