From d0a2b2ec77ade899d23da54509c0661c239e86cc Mon Sep 17 00:00:00 2001 From: LeSingh1 Date: Sat, 8 Aug 2026 18:05:36 -0700 Subject: [PATCH] Recover from the nvidia-cufile "undefined symbol: shm_open" load failure nvidia-cufile wheels reference shm_open() but carry no DT_NEEDED entry for librt. On glibc < 2.34, where shm_open still lives in librt rather than in libc, dlopen() therefore fails and load_nvidia_dynamic_lib("cufile") raises: RuntimeError: Failed to dlopen .../nvidia/cu13/lib/libcufile.so.0: .../libcufile.so.0: undefined symbol: shm_open Loading librt.so.1 with RTLD_GLOBAL beforehand makes the reference resolve, which is the workaround kvikio carries today (rapidsai/kvikio#1000). Apply it as a strictly reactive retry rather than an unconditional preload: nothing happens unless the primary load has already failed with exactly this signature (libname "cufile" plus "shm_open" in the dlopen error). That keeps the normal path byte-for-byte unchanged, keeps librt out of the global symbol scope for everyone else, and makes the workaround disappear by itself once the wheel is fixed. If the retry does not help, the original error is reported, since it is the more informative one. Adds tests/test_load_dl_linux.py covering the retry, the two ways it can give up, and that neither an unrelated cufile failure nor the same symptom on a different library triggers it. Refs #2313 --- .../pathfinder/_dynamic_libs/load_dl_linux.py | 34 ++++- cuda_pathfinder/tests/test_load_dl_linux.py | 135 ++++++++++++++++++ 2 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 cuda_pathfinder/tests/test_load_dl_linux.py diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/load_dl_linux.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/load_dl_linux.py index 24d4d67c9e2..98bcf275be4 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/load_dl_linux.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/load_dl_linux.py @@ -208,6 +208,35 @@ def _work_around_known_bugs(libname: str, found_path: str) -> None: ctypes.CDLL(dep_path, CDLL_MODE) +def _retry_load_after_working_around_known_bugs( + desc: LibDescriptor, found_path: str, error: OSError +) -> ctypes.CDLL | None: + """Second-chance load for known-broken NVIDIA wheels. + + Returns the loaded library, or None if no workaround applies or the retry + also fails. Kept strictly reactive: nothing here runs unless the primary + load already failed with the exact signature of the known bug, so it + disappears by itself once the offending wheel is fixed. + """ + if desc.name != "cufile" or "shm_open" not in str(error): + return None + # Work around bug in nvidia-cufile wheels (observed with + # nvidia_cufile_cu13-1.15.1.6-py3-none-manylinux_2_28_aarch64.whl) + # Issue: libcufile.so.0 references shm_open() but has no DT_NEEDED entry + # for librt, so dlopen() fails with "undefined symbol: shm_open" on + # glibc < 2.34, where shm_open lives in librt rather than in libc. + # Making librt's symbols globally visible first lets the reference resolve. + # See https://github.com/NVIDIA/cuda-python/issues/2313 + try: + ctypes.CDLL("librt.so.1", CDLL_MODE) + except OSError: + return None # Nothing gained; defer to the original error. + try: + return _load_lib(desc, found_path) + except OSError: + return None # Defer to the original error, which is the more informative one. + + def load_with_abs_path(desc: LibDescriptor, found_path: str, found_via: str | None = None) -> LoadedDL: """Load a dynamic library from the given path. @@ -226,5 +255,8 @@ def load_with_abs_path(desc: LibDescriptor, found_path: str, found_via: str | No try: handle = _load_lib(desc, found_path) except OSError as e: - raise RuntimeError(f"Failed to dlopen {found_path}: {e}") from e + retried_handle = _retry_load_after_working_around_known_bugs(desc, found_path, e) + if retried_handle is None: + raise RuntimeError(f"Failed to dlopen {found_path}: {e}") from e + handle = retried_handle return LoadedDL(found_path, False, handle._handle, found_via) diff --git a/cuda_pathfinder/tests/test_load_dl_linux.py b/cuda_pathfinder/tests/test_load_dl_linux.py new file mode 100644 index 00000000000..79689f5d519 --- /dev/null +++ b/cuda_pathfinder/tests/test_load_dl_linux.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import sys + +import pytest + +from cuda.pathfinder._dynamic_libs.descriptor_catalog import DescriptorSpec + +# load_dl_linux binds dlinfo()/dlerror() from libdl at module scope, which only +# exists off Windows. platform_loader.py routes every non-win32 platform here. +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="Exercises the non-Windows dynamic library loader") + +_SHM_OPEN_ERROR = "libcufile.so.0: undefined symbol: shm_open" + +CUFILE_DESC = DescriptorSpec( + name="cufile", + packaged_with="ctk", + linux_sonames=("libcufile.so.0",), +) + + +class FakeHandle: + _handle = 0x1234 + + +def _record_cdll_calls(monkeypatch, load_dl_linux, *, librt_loadable: bool = True) -> list[str]: + """Capture ctypes.CDLL() calls made by the module under test.""" + loaded: list[str] = [] + + def fake_cdll(name, *args, **kwargs): + loaded.append(name) + if not librt_loadable: + raise OSError(f"{name}: cannot open shared object file") + return FakeHandle() + + monkeypatch.setattr(load_dl_linux.ctypes, "CDLL", fake_cdll) + return loaded + + +@pytest.mark.agent_authored(model="claude-opus-5") +def test_cufile_shm_open_failure_is_retried_after_loading_librt(monkeypatch): + """nvidia-cufile wheels reference shm_open() without a DT_NEEDED entry for librt. + + See https://github.com/NVIDIA/cuda-python/issues/2313: on glibc < 2.34 + shm_open lives in librt, so the first dlopen() fails and only succeeds + once librt's symbols are globally visible. + """ + from cuda.pathfinder._dynamic_libs import load_dl_linux + + loaded = _record_cdll_calls(monkeypatch, load_dl_linux) + attempts: list[str] = [] + + def fake_load_lib(desc, filename): + attempts.append(filename) + if len(attempts) == 1: + raise OSError(_SHM_OPEN_ERROR) + return FakeHandle() + + monkeypatch.setattr(load_dl_linux, "_load_lib", fake_load_lib) + + loaded_dl = load_dl_linux.load_with_abs_path(CUFILE_DESC, "/site-packages/nvidia/cu13/lib/libcufile.so.0", "conda") + + assert loaded_dl.abs_path == "/site-packages/nvidia/cu13/lib/libcufile.so.0" + assert loaded_dl.found_via == "conda" + assert loaded_dl._handle_uint == FakeHandle._handle + assert loaded == ["librt.so.1"] + assert len(attempts) == 2 + + +@pytest.mark.agent_authored(model="claude-opus-5") +def test_cufile_retry_reports_the_original_error_if_librt_does_not_help(monkeypatch): + from cuda.pathfinder._dynamic_libs import load_dl_linux + + loaded = _record_cdll_calls(monkeypatch, load_dl_linux) + + def fake_load_lib(desc, filename): + raise OSError(_SHM_OPEN_ERROR) + + monkeypatch.setattr(load_dl_linux, "_load_lib", fake_load_lib) + + with pytest.raises(RuntimeError, match="undefined symbol: shm_open"): + load_dl_linux.load_with_abs_path(CUFILE_DESC, "/site-packages/nvidia/cu13/lib/libcufile.so.0") + + assert loaded == ["librt.so.1"] + + +@pytest.mark.agent_authored(model="claude-opus-5") +def test_cufile_retry_is_skipped_if_librt_cannot_be_loaded(monkeypatch): + from cuda.pathfinder._dynamic_libs import load_dl_linux + + loaded = _record_cdll_calls(monkeypatch, load_dl_linux, librt_loadable=False) + attempts: list[str] = [] + + def fake_load_lib(desc, filename): + attempts.append(filename) + raise OSError(_SHM_OPEN_ERROR) + + monkeypatch.setattr(load_dl_linux, "_load_lib", fake_load_lib) + + with pytest.raises(RuntimeError, match="undefined symbol: shm_open"): + load_dl_linux.load_with_abs_path(CUFILE_DESC, "/site-packages/nvidia/cu13/lib/libcufile.so.0") + + assert loaded == ["librt.so.1"] + assert len(attempts) == 1 + + +@pytest.mark.agent_authored(model="claude-opus-5") +@pytest.mark.parametrize( + ("libname", "message"), + [ + ("cufile", "libcufile.so.0: cannot open shared object file"), # unrelated cufile failure + ("cublas", _SHM_OPEN_ERROR), # right symptom, wrong library + ], +) +def test_unrelated_dlopen_failures_do_not_trigger_the_librt_workaround(monkeypatch, libname, message): + from cuda.pathfinder._dynamic_libs import load_dl_linux + + loaded = _record_cdll_calls(monkeypatch, load_dl_linux) + attempts: list[str] = [] + + def fake_load_lib(desc, filename): + attempts.append(filename) + raise OSError(message) + + monkeypatch.setattr(load_dl_linux, "_load_lib", fake_load_lib) + + desc = DescriptorSpec(name=libname, packaged_with="ctk", linux_sonames=(f"lib{libname}.so.0",)) + with pytest.raises(RuntimeError, match="Failed to dlopen"): + load_dl_linux.load_with_abs_path(desc, f"/site-packages/lib{libname}.so.0") + + assert loaded == [] + assert len(attempts) == 1