diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py index d9567207e..75ebec3a8 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import functools import glob import os from collections.abc import Sequence @@ -9,13 +8,13 @@ from cuda.pathfinder._dynamic_libs.load_dl_common import DynamicLibNotFoundError from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import ( - IS_WINDOWS, SITE_PACKAGES_LIBDIRS_LINUX, SITE_PACKAGES_LIBDIRS_WINDOWS, is_suppressed_dll_file, ) from cuda.pathfinder._utils.env_vars import get_cuda_home_or_path -from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs, find_sub_dirs_all_sitepackages +from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS def _no_such_file_in_sub_dirs( @@ -80,37 +79,50 @@ def _find_dll_using_nvidia_bin_dirs( return None -def _find_lib_dir_using_cuda_home(libname: str) -> Optional[str]: - cuda_home = get_cuda_home_or_path() - if cuda_home is None: - return None - subdirs_list: tuple[tuple[str, ...], ...] +def _find_lib_dir_using_anchor_point(libname: str, anchor_point: str, linux_lib_dir: str) -> Optional[str]: + # Resolve paths for the four cases: + # Windows/Linux x nvvm yes/no if IS_WINDOWS: if libname == "nvvm": # noqa: SIM108 - subdirs_list = ( - ("nvvm", "bin", "*"), # CTK 13 - ("nvvm", "bin"), # CTK 12 - ) + rel_paths = [ + "nvvm/bin/*", # CTK 13 + "nvvm/bin", # CTK 12 + ] else: - subdirs_list = ( - ("bin", "x64"), # CTK 13 - ("bin",), # CTK 12 - ) + rel_paths = [ + "bin/x64", # CTK 13 + "bin", # CTK 12 + ] else: if libname == "nvvm": # noqa: SIM108 - subdirs_list = (("nvvm", "lib64"),) + rel_paths = ["nvvm/lib64"] else: - subdirs_list = ( - ("lib64",), # CTK - ("lib",), # Conda - ) - for sub_dirs in subdirs_list: - dirname: str # work around bug in mypy - for dirname in find_sub_dirs((cuda_home,), sub_dirs): - return dirname + rel_paths = [linux_lib_dir] + + for rel_path in rel_paths: + for dirname in sorted(glob.glob(os.path.join(anchor_point, rel_path))): + if os.path.isdir(dirname): + return dirname + return None +def _find_lib_dir_using_cuda_home(libname: str) -> Optional[str]: + cuda_home = get_cuda_home_or_path() + if cuda_home is None: + return None + return _find_lib_dir_using_anchor_point(libname, anchor_point=cuda_home, linux_lib_dir="lib64") + + +def _find_lib_dir_using_conda_prefix(libname: str) -> Optional[str]: + conda_prefix = os.environ.get("CONDA_PREFIX") + if not conda_prefix: + return None + return _find_lib_dir_using_anchor_point( + libname, anchor_point=os.path.join(conda_prefix, "Library") if IS_WINDOWS else conda_prefix, linux_lib_dir="lib" + ) + + def _find_so_using_lib_dir( lib_dir: str, so_basename: str, error_messages: list[str], attachments: list[str] ) -> Optional[str]: @@ -144,55 +156,55 @@ def _find_dll_using_lib_dir( class _FindNvidiaDynamicLib: def __init__(self, libname: str): self.libname = libname + if IS_WINDOWS: + self.lib_searched_for = f"{libname}*.dll" + else: + self.lib_searched_for = f"lib{libname}.so" self.error_messages: list[str] = [] self.attachments: list[str] = [] - self.abs_path = None + self.abs_path: Optional[str] = None + def try_site_packages(self) -> Optional[str]: if IS_WINDOWS: - self.lib_searched_for = f"{libname}*.dll" - if self.abs_path is None: - self.abs_path = _find_dll_using_nvidia_bin_dirs( - libname, - self.lib_searched_for, - self.error_messages, - self.attachments, - ) + return _find_dll_using_nvidia_bin_dirs( + self.libname, + self.lib_searched_for, + self.error_messages, + self.attachments, + ) else: - self.lib_searched_for = f"lib{libname}.so" - if self.abs_path is None: - self.abs_path = _find_so_using_nvidia_lib_dirs( - libname, - self.lib_searched_for, - self.error_messages, - self.attachments, - ) - - def retry_with_cuda_home_priority_last(self) -> None: - cuda_home_lib_dir = _find_lib_dir_using_cuda_home(self.libname) - if cuda_home_lib_dir is not None: - if IS_WINDOWS: - self.abs_path = _find_dll_using_lib_dir( - cuda_home_lib_dir, - self.libname, - self.error_messages, - self.attachments, - ) - else: - self.abs_path = _find_so_using_lib_dir( - cuda_home_lib_dir, - self.lib_searched_for, - self.error_messages, - self.attachments, - ) - - def raise_if_abs_path_is_None(self) -> str: # noqa: N802 - if self.abs_path: - return self.abs_path + return _find_so_using_nvidia_lib_dirs( + self.libname, + self.lib_searched_for, + self.error_messages, + self.attachments, + ) + + def try_with_conda_prefix(self) -> Optional[str]: + return self._find_using_lib_dir(_find_lib_dir_using_conda_prefix(self.libname)) + + def try_with_cuda_home(self) -> Optional[str]: + return self._find_using_lib_dir(_find_lib_dir_using_cuda_home(self.libname)) + + def _find_using_lib_dir(self, lib_dir: Optional[str]) -> Optional[str]: + if lib_dir is None: + return None + if IS_WINDOWS: + return _find_dll_using_lib_dir( + lib_dir, + self.libname, + self.error_messages, + self.attachments, + ) + else: + return _find_so_using_lib_dir( + lib_dir, + self.lib_searched_for, + self.error_messages, + self.attachments, + ) + + def raise_not_found_error(self) -> None: err = ", ".join(self.error_messages) att = "\n".join(self.attachments) raise DynamicLibNotFoundError(f'Failure finding "{self.lib_searched_for}": {err}\n{att}') - - -@functools.cache -def find_nvidia_dynamic_lib(libname: str) -> str: - return _FindNvidiaDynamicLib(libname).raise_if_abs_path_is_None() diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/load_nvidia_dynamic_lib.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/load_nvidia_dynamic_lib.py index 3160333aa..6a8392589 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/load_nvidia_dynamic_lib.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/load_nvidia_dynamic_lib.py @@ -7,7 +7,7 @@ from cuda.pathfinder._dynamic_libs.find_nvidia_dynamic_lib import _FindNvidiaDynamicLib from cuda.pathfinder._dynamic_libs.load_dl_common import LoadedDL, load_dependencies -from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import IS_WINDOWS +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS if IS_WINDOWS: from cuda.pathfinder._dynamic_libs.load_dl_windows import ( @@ -24,13 +24,15 @@ def _load_lib_no_cache(libname: str) -> LoadedDL: - found = _FindNvidiaDynamicLib(libname) - have_abs_path = found.abs_path is not None + finder = _FindNvidiaDynamicLib(libname) + abs_path = finder.try_site_packages() + if abs_path is None: + abs_path = finder.try_with_conda_prefix() # If the library was already loaded by someone else, reproduce any OS-specific # side-effects we would have applied on a direct absolute-path load (e.g., # AddDllDirectory on Windows for libs that require it). - loaded = check_if_already_loaded_from_elsewhere(libname, have_abs_path) + loaded = check_if_already_loaded_from_elsewhere(libname, abs_path is not None) # Load dependencies regardless of who loaded the primary lib first. # Doing this *after* the side-effect ensures dependencies resolve consistently @@ -40,15 +42,15 @@ def _load_lib_no_cache(libname: str) -> LoadedDL: if loaded is not None: return loaded - if not have_abs_path: + if abs_path is None: loaded = load_with_system_search(libname) if loaded is not None: return loaded - found.retry_with_cuda_home_priority_last() - found.raise_if_abs_path_is_None() + abs_path = finder.try_with_cuda_home() + if abs_path is None: + finder.raise_not_found_error() - assert found.abs_path is not None # for mypy - return load_with_abs_path(libname, found.abs_path) + return load_with_abs_path(libname, abs_path) @functools.cache @@ -62,6 +64,18 @@ def load_nvidia_dynamic_lib(libname: str) -> LoadedDL: Returns: LoadedDL: Object containing the OS library handle and absolute path. + **Important:** + + **Never close the returned handle.** Do **not** call ``dlclose`` (Linux) or + ``FreeLibrary`` (Windows) on the ``LoadedDL._handle_uint``. + + **Why:** the return value is cached (``functools.cache``) and shared across the + process. Closing the handle can unload the module while other code still uses + it, leading to crashes or subtle failures. + + This applies to Linux and Windows. For context, see issue #1011: + https://github.com/NVIDIA/cuda-python/issues/1011 + Raises: DynamicLibNotFoundError: If the library cannot be found or loaded. RuntimeError: If Python is not 64-bit. @@ -77,7 +91,13 @@ def load_nvidia_dynamic_lib(libname: str) -> LoadedDL: - Scan installed distributions (``site-packages``) to find libraries shipped in NVIDIA wheels. - 2. **OS default mechanisms / Conda environments** + 2. **Conda environment** + + - Conda installations are discovered via ``CONDA_PREFIX``, which is + defined automatically in activated conda environments (see + https://docs.conda.io/projects/conda-build/en/stable/user-guide/environment-variables.html). + + 3. **OS default mechanisms** - Fall back to the native loader: @@ -85,14 +105,6 @@ def load_nvidia_dynamic_lib(libname: str) -> LoadedDL: - Windows: ``LoadLibraryW()`` - - Conda installations are commonly discovered via: - - - Linux: ``$ORIGIN/../lib`` in the ``RPATH`` of the ``python`` binary - (note: this can take precedence over ``LD_LIBRARY_PATH`` and - ``/etc/ld.so.conf.d/``). - - - Windows: ``%CONDA_PREFIX%\\Library\\bin`` on the system ``PATH``. - - CUDA Toolkit (CTK) system installs with system config updates are often discovered via: @@ -101,7 +113,7 @@ def load_nvidia_dynamic_lib(libname: str) -> LoadedDL: - Windows: ``C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\vX.Y\\bin`` on the system ``PATH``. - 3. **Environment variables** + 4. **Environment variables** - If set, use ``CUDA_HOME`` or ``CUDA_PATH`` (in that order). diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py index 4b1eb5ce6..7933e909a 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py @@ -7,9 +7,7 @@ # SUPPORTED_WINDOWS_DLLS # SUPPORTED_LINUX_SONAMES -import sys - -IS_WINDOWS = sys.platform == "win32" +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS SUPPORTED_LIBNAMES_COMMON = ( # Core CUDA Runtime and Compiler diff --git a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py index f97f12c06..535d4b800 100644 --- a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py +++ b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py @@ -7,9 +7,9 @@ from typing import Optional from cuda.pathfinder._headers import supported_nvidia_headers -from cuda.pathfinder._headers.supported_nvidia_headers import IS_WINDOWS from cuda.pathfinder._utils.env_vars import get_cuda_home_or_path from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS def _abs_norm(path: Optional[str]) -> Optional[str]: @@ -90,7 +90,7 @@ def _find_ctk_header_directory(libname: str) -> Optional[str]: if _joined_isfile(hdr_dir, h_basename): return hdr_dir - conda_prefix = os.getenv("CONDA_PREFIX") + conda_prefix = os.environ.get("CONDA_PREFIX") if conda_prefix: # noqa: SIM102 if result := _find_based_on_conda_layout(libname, h_basename, conda_prefix): return result diff --git a/cuda_pathfinder/cuda/pathfinder/_headers/supported_nvidia_headers.py b/cuda_pathfinder/cuda/pathfinder/_headers/supported_nvidia_headers.py index afd9067de..1b2855c0c 100644 --- a/cuda_pathfinder/cuda/pathfinder/_headers/supported_nvidia_headers.py +++ b/cuda_pathfinder/cuda/pathfinder/_headers/supported_nvidia_headers.py @@ -1,10 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import sys from typing import Final -IS_WINDOWS = sys.platform == "win32" +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS SUPPORTED_HEADERS_CTK_COMMON = { "cccl": "cuda/std/version", diff --git a/cuda_pathfinder/cuda/pathfinder/_utils/platform_aware.py b/cuda_pathfinder/cuda/pathfinder/_utils/platform_aware.py new file mode 100644 index 000000000..196a0df7f --- /dev/null +++ b/cuda_pathfinder/cuda/pathfinder/_utils/platform_aware.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import sys + +IS_WINDOWS = sys.platform == "win32" + + +def quote_for_shell(s: str) -> str: + if IS_WINDOWS: + # This is a relatively heavy import; keep pathfinder if possible. + from subprocess import list2cmdline # nosec B404 + + return list2cmdline([s]) + else: + import shlex + + return shlex.quote(s) diff --git a/cuda_pathfinder/cuda/pathfinder/_version.py b/cuda_pathfinder/cuda/pathfinder/_version.py index 001da9389..9fca58d64 100644 --- a/cuda_pathfinder/cuda/pathfinder/_version.py +++ b/cuda_pathfinder/cuda/pathfinder/_version.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -__version__ = "1.2.3" +__version__ = "1.3.0" diff --git a/cuda_pathfinder/docs/nv-versions.json b/cuda_pathfinder/docs/nv-versions.json index 9fcc3f0ab..db8fecd82 100644 --- a/cuda_pathfinder/docs/nv-versions.json +++ b/cuda_pathfinder/docs/nv-versions.json @@ -3,6 +3,10 @@ "version": "latest", "url": "https://nvidia.github.io/cuda-python/cuda-pathfinder/latest/" }, + { + "version": "1.3.0", + "url": "https://nvidia.github.io/cuda-python/cuda-pathfinder/1.3.0/" + }, { "version": "1.2.3", "url": "https://nvidia.github.io/cuda-python/cuda-pathfinder/1.2.3/" diff --git a/cuda_pathfinder/docs/source/conf.py b/cuda_pathfinder/docs/source/conf.py index 4ede571f2..1ef35cc18 100644 --- a/cuda_pathfinder/docs/source/conf.py +++ b/cuda_pathfinder/docs/source/conf.py @@ -85,7 +85,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +html_static_path = [] # ["_static"] does not exist in our environment # skip cmdline prompts copybutton_exclude = ".linenos, .gp" diff --git a/cuda_pathfinder/docs/source/contribute.rst b/cuda_pathfinder/docs/source/contribute.rst index 4bfcd9c38..26b592ca5 100644 --- a/cuda_pathfinder/docs/source/contribute.rst +++ b/cuda_pathfinder/docs/source/contribute.rst @@ -11,7 +11,7 @@ Thank you for your interest in contributing to ``cuda-pathfinder``! Based on the 1. You want to report a bug, feature request, or documentation issue - File an `issue `_ describing what you encountered or what you want to see changed. - The NVIDIA team will evaluate the issues and triage them, scheduling - them for a release. If you believe the issue needs priority attention - comment on the issue to notify the team. -2. You want to implement a feature, improvement, or bug fix: - - Please ensure that your commits are signed `following GitHub's instruction `_. + them for a release. If you believe the issue needs priority attention + comment on the issue to notify the team. +2. You want to implement a feature, improvement, or bug fix + - Please ensure that your commits are signed `following GitHub's instruction `_. diff --git a/cuda_pathfinder/docs/source/release.rst b/cuda_pathfinder/docs/source/release.rst index 62dbf7ad6..b46579557 100644 --- a/cuda_pathfinder/docs/source/release.rst +++ b/cuda_pathfinder/docs/source/release.rst @@ -7,6 +7,7 @@ Release Notes .. toctree:: :maxdepth: 3 + 1.3.0 1.2.3 1.2.2 1.2.1 diff --git a/cuda_pathfinder/docs/source/release/1.0.0-notes.rst b/cuda_pathfinder/docs/source/release/1.0.0-notes.rst index 33c794197..a8cfc3857 100644 --- a/cuda_pathfinder/docs/source/release/1.0.0-notes.rst +++ b/cuda_pathfinder/docs/source/release/1.0.0-notes.rst @@ -1,7 +1,7 @@ .. SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 -.. module:: cuda.pathfinder +.. py:currentmodule:: cuda.pathfinder ``cuda-pathfinder`` 1.0.0 Release notes ======================================== diff --git a/cuda_pathfinder/docs/source/release/1.1.0-notes.rst b/cuda_pathfinder/docs/source/release/1.1.0-notes.rst index 14c3ba5b8..dd09179f3 100644 --- a/cuda_pathfinder/docs/source/release/1.1.0-notes.rst +++ b/cuda_pathfinder/docs/source/release/1.1.0-notes.rst @@ -1,7 +1,7 @@ .. SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 -.. module:: cuda.pathfinder +.. py:currentmodule:: cuda.pathfinder ``cuda-pathfinder`` 1.1.0 Release notes ======================================== diff --git a/cuda_pathfinder/docs/source/release/1.2.0-notes.rst b/cuda_pathfinder/docs/source/release/1.2.0-notes.rst index 6037dba84..56a40cf59 100644 --- a/cuda_pathfinder/docs/source/release/1.2.0-notes.rst +++ b/cuda_pathfinder/docs/source/release/1.2.0-notes.rst @@ -1,7 +1,7 @@ .. SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 -.. module:: cuda.pathfinder +.. py:currentmodule:: cuda.pathfinder ``cuda-pathfinder`` 1.2.0 Release notes ======================================== diff --git a/cuda_pathfinder/docs/source/release/1.2.1-notes.rst b/cuda_pathfinder/docs/source/release/1.2.1-notes.rst index 7a8e410fe..fe0e147c1 100644 --- a/cuda_pathfinder/docs/source/release/1.2.1-notes.rst +++ b/cuda_pathfinder/docs/source/release/1.2.1-notes.rst @@ -1,7 +1,7 @@ .. SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 -.. module:: cuda.pathfinder +.. py:currentmodule:: cuda.pathfinder ``cuda-pathfinder`` 1.2.1 Release notes ======================================= diff --git a/cuda_pathfinder/docs/source/release/1.2.2-notes.rst b/cuda_pathfinder/docs/source/release/1.2.2-notes.rst index 0a483081e..b879fb23a 100644 --- a/cuda_pathfinder/docs/source/release/1.2.2-notes.rst +++ b/cuda_pathfinder/docs/source/release/1.2.2-notes.rst @@ -1,7 +1,7 @@ .. SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 -.. module:: cuda.pathfinder +.. py:currentmodule:: cuda.pathfinder ``cuda-pathfinder`` 1.2.2 Release notes ======================================= diff --git a/cuda_pathfinder/docs/source/release/1.2.3-notes.rst b/cuda_pathfinder/docs/source/release/1.2.3-notes.rst index 3fa08bd19..18e146906 100644 --- a/cuda_pathfinder/docs/source/release/1.2.3-notes.rst +++ b/cuda_pathfinder/docs/source/release/1.2.3-notes.rst @@ -1,7 +1,7 @@ .. SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 -.. module:: cuda.pathfinder +.. py:currentmodule:: cuda.pathfinder ``cuda-pathfinder`` 1.2.3 Release notes ======================================= diff --git a/cuda_pathfinder/docs/source/release/1.3.0-notes.rst b/cuda_pathfinder/docs/source/release/1.3.0-notes.rst new file mode 100644 index 000000000..91b0ac732 --- /dev/null +++ b/cuda_pathfinder/docs/source/release/1.3.0-notes.rst @@ -0,0 +1,33 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +.. py:currentmodule:: cuda.pathfinder + +``cuda-pathfinder`` 1.3.0 Release notes +======================================= + +Released on Sep 29, 2025 + +Important update to the ``load_nvidia_dynamic_lib`` documentation +----------------------------------------------------------------- + +The documentation for :func:`cuda.pathfinder.load_nvidia_dynamic_lib` now calls out +an important behavior: + +* **Do not close** the OS handle returned (``LoadedDL._handle_uint``). Avoid + ``dlclose`` (Linux) and ``FreeLibrary`` (Windows). + +* **Reason:** the function's result is cached (``functools.cache``) and shared across + the process. Closing the handle can unload the module while other code still holds + a reference, which may cause crashes or latent errors. + +This note is documentation‑only; there is no API change in 1.3.0. See issue #1011 +for background and discussion: +https://github.com/NVIDIA/cuda-python/issues/1011 + +Highlights +---------- + +* Add a Conda-specific search into ``load_nvidia_dynamic_lib()``. + This makes the search more intuitive and predictable. + (`PR #1003 `_) diff --git a/cuda_pathfinder/tests/child_load_nvidia_dynamic_lib_helper.py b/cuda_pathfinder/tests/child_load_nvidia_dynamic_lib_helper.py index 4ca905989..72ee80c89 100644 --- a/cuda_pathfinder/tests/child_load_nvidia_dynamic_lib_helper.py +++ b/cuda_pathfinder/tests/child_load_nvidia_dynamic_lib_helper.py @@ -5,8 +5,10 @@ # lightweight module. That avoids re-importing the test module (and # repeating its potentially expensive setup) in every child process. +import json import os import sys +import traceback def build_child_process_failed_for_libname_message(libname, result): @@ -24,15 +26,20 @@ def validate_abs_path(abs_path): def child_process_func(libname): - from cuda.pathfinder import load_nvidia_dynamic_lib + from cuda.pathfinder import DynamicLibNotFoundError, load_nvidia_dynamic_lib from cuda.pathfinder._dynamic_libs.load_nvidia_dynamic_lib import _load_lib_no_cache from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import ( - IS_WINDOWS, SUPPORTED_LINUX_SONAMES, SUPPORTED_WINDOWS_DLLS, ) - - loaded_dl_fresh = load_nvidia_dynamic_lib(libname) + from cuda.pathfinder._utils.platform_aware import IS_WINDOWS + + try: + loaded_dl_fresh = load_nvidia_dynamic_lib(libname) + except DynamicLibNotFoundError: + print("CHILD_LOAD_NVIDIA_DYNAMIC_LIB_HELPER_DYNAMIC_LIB_NOT_FOUND_ERROR:") # noqa: T201 + traceback.print_exc(file=sys.stdout) + return if loaded_dl_fresh.was_already_loaded_from_elsewhere: raise RuntimeError("loaded_dl_fresh.was_already_loaded_from_elsewhere") validate_abs_path(loaded_dl_fresh.abs_path) @@ -50,4 +57,4 @@ def child_process_func(libname): raise RuntimeError(f"not os.path.samefile({loaded_dl_no_cache.abs_path=!r}, {loaded_dl_fresh.abs_path=!r})") validate_abs_path(loaded_dl_no_cache.abs_path) - sys.stdout.write(f"{loaded_dl_fresh.abs_path!r}\n") + print(json.dumps(loaded_dl_fresh.abs_path)) # noqa: T201 diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index da0f0e01e..cdea0cd28 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -22,11 +22,11 @@ from cuda.pathfinder import find_nvidia_header_directory from cuda.pathfinder._headers.supported_nvidia_headers import ( - IS_WINDOWS, SUPPORTED_HEADERS_CTK, SUPPORTED_HEADERS_CTK_ALL, SUPPORTED_SITE_PACKAGE_HEADER_DIRS_CTK, ) +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS STRICTNESS = os.environ.get("CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS", "see_what_works") assert STRICTNESS in ("see_what_works", "all_must_work") @@ -59,7 +59,7 @@ def test_find_libname_nvshmem(info_summary_append): if have_nvidia_nvshmem_package(): hdr_dir_parts = hdr_dir.split(os.path.sep) assert "site-packages" in hdr_dir_parts - elif conda_prefix := os.getenv("CONDA_PREFIX"): + elif conda_prefix := os.environ.get("CONDA_PREFIX"): assert hdr_dir.startswith(conda_prefix) else: assert hdr_dir.startswith("/usr/include/nvshmem_") diff --git a/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib.py b/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib.py index 5f35d996d..b50c644a6 100644 --- a/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib.py +++ b/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import functools +import json import os from unittest.mock import patch @@ -13,6 +14,7 @@ from cuda.pathfinder._dynamic_libs import supported_nvidia_libs from cuda.pathfinder._utils.find_site_packages_dll import find_all_dll_files_via_metadata from cuda.pathfinder._utils.find_site_packages_so import find_all_so_files_via_metadata +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS, quote_for_shell STRICTNESS = os.environ.get("CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS", "see_what_works") assert STRICTNESS in ("see_what_works", "all_must_work") @@ -72,7 +74,7 @@ def test_runtime_error_on_non_64bit_python(): @functools.cache def _get_libnames_for_test_load_nvidia_dynamic_lib(): result = list(SUPPORTED_NVIDIA_LIBNAMES) - if supported_nvidia_libs.IS_WINDOWS: + if IS_WINDOWS: spld_other = supported_nvidia_libs.SITE_PACKAGES_LIBDIRS_WINDOWS_OTHER all_dyn_libs = find_all_dll_files_via_metadata() for libname in spld_other: @@ -96,11 +98,20 @@ def test_load_nvidia_dynamic_lib(info_summary_append, libname): # to ensure isolation of global dynamic linking state (e.g., dlopen handles). # Without child processes, loading/unloading libraries during testing could # interfere across test cases and lead to nondeterministic or platform-specific failures. - timeout = 120 if supported_nvidia_libs.IS_WINDOWS else 30 + timeout = 120 if IS_WINDOWS else 30 result = spawned_process_runner.run_in_spawned_child_process(child_process_func, args=(libname,), timeout=timeout) - if result.returncode == 0: - info_summary_append(f"abs_path={result.stdout.rstrip()}") - elif STRICTNESS == "see_what_works" or "DynamicLibNotFoundError: Failure finding " in result.stderr: + + def raise_child_process_failed(): + raise RuntimeError(build_child_process_failed_for_libname_message(libname, result)) + + if result.returncode != 0: + raise_child_process_failed() + assert not result.stderr + if result.stdout.startswith("CHILD_LOAD_NVIDIA_DYNAMIC_LIB_HELPER_DYNAMIC_LIB_NOT_FOUND_ERROR:"): + if STRICTNESS == "all_must_work": + raise_child_process_failed() info_summary_append(f"Not found: {libname=!r}") else: - raise RuntimeError(build_child_process_failed_for_libname_message(libname, result)) + abs_path = json.loads(result.stdout.rstrip()) + info_summary_append(f"abs_path={quote_for_shell(abs_path)}") + assert os.path.isfile(abs_path) # double-check the abs_path