diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c12694..3307578 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: - uses: astral-sh/setup-uv@v9.0.0 - name: Test package - run: uv run --with "clang<19" --group test pytest --forked + run: uv run --with "clang<19" --group test pytest # Commented for now -- msys2 Clang (v15) and the clang Python package (v14) are incompatible # diff --git a/pybind11_mkdoc/mkdoc_lib.py b/pybind11_mkdoc/mkdoc_lib.py index 4b4ab34..0a553d7 100755 --- a/pybind11_mkdoc/mkdoc_lib.py +++ b/pybind11_mkdoc/mkdoc_lib.py @@ -15,9 +15,9 @@ import sys import textwrap from collections import OrderedDict +from concurrent.futures import ThreadPoolExecutor from glob import glob -from multiprocessing import cpu_count -from threading import Semaphore, Thread +from itertools import repeat from clang import cindex from clang.cindex import CursorKind @@ -94,9 +94,6 @@ CPP_OPERATORS = OrderedDict(sorted(CPP_OPERATORS.items(), key=lambda t: -len(t[0]))) -job_count = cpu_count() -job_semaphore = Semaphore(job_count) -errors_detected = False docstring_width = 70 @@ -555,25 +552,12 @@ def extract(filename, node, prefix, output, file_cache): return None -class ExtractionThread(Thread): - def __init__(self, filename, parameters, output): - Thread.__init__(self) - self.filename = filename - self.parameters = parameters - self.output = output - job_semaphore.acquire() - - def run(self): - global errors_detected - try: - index = cindex.Index(cindex.conf.lib.clang_createIndex(False, True)) - tu = index.parse(self.filename, self.parameters) - extract(self.filename, tu.cursor, "", self.output, {}) - except BaseException: - errors_detected = True - raise - finally: - job_semaphore.release() +def _extract_file(filename, parameters): + index = cindex.Index(cindex.conf.lib.clang_createIndex(False, True)) + tu = index.parse(filename, parameters) + output = [] + extract(filename, tu.cursor, "", output, {}) + return output def read_args(args): @@ -591,7 +575,8 @@ def read_args(args): sdk_dir = dev_path + "Platforms/MacOSX.platform/Developer/SDKs" libclang = lib_dir + "libclang.dylib" - if os.path.exists(libclang): + # cindex forbids (re)configuring the library once it has been loaded + if os.path.exists(libclang) and not cindex.Config.loaded: cindex.Config.set_library_path(os.path.dirname(libclang)) if os.path.exists(sdk_dir): @@ -602,13 +587,14 @@ def read_args(args): if "LIBCLANG_PATH" in os.environ: library_file = os.environ["LIBCLANG_PATH"] if os.path.isfile(library_file): - cindex.Config.set_library_file(library_file) + if not cindex.Config.loaded: + cindex.Config.set_library_file(library_file) else: msg = "Failed to find libclang.dll! Set the LIBCLANG_PATH environment variable to provide a path to it." raise FileNotFoundError(msg) else: library_file = ctypes.util.find_library("libclang.dll") - if library_file is not None: + if library_file is not None and not cindex.Config.loaded: cindex.Config.set_library_file(library_file) elif platform.system() == "Linux": # LLVM switched to a monolithical setup that includes everything under @@ -650,7 +636,8 @@ def folder_version(d): else: libclang_dir = os.path.join(llvm_dir, "lib", "libclang.so.1") - cindex.Config.set_library_file(libclang_dir) + if not cindex.Config.loaded: + cindex.Config.set_library_file(libclang_dir) cpp_dirs = [] if "-stdlib=libc++" not in args: @@ -696,15 +683,9 @@ def folder_version(d): def extract_all(args): parameters, filenames = read_args(args) - output = [] - for filename in filenames: - thr = ExtractionThread(filename, parameters, output) - thr.start() - - for _i in range(job_count): - job_semaphore.acquire() - - return output + with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: + results = executor.map(_extract_file, filenames, repeat(parameters)) + return [comment for output in results for comment in output] def write_header(comments, out_file=sys.stdout): @@ -765,8 +746,6 @@ def mkdoc(args, width, output=None): global docstring_width docstring_width = int(width) comments = extract_all(args) - if errors_detected: - return if output: try: diff --git a/pyproject.toml b/pyproject.toml index 06f89af..e1cb809 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ dynamic = ["version"] test = [ "pytest", "pybind11", - "pytest-forked", ] dev = [{ include-group = "test" }] diff --git a/tests/cli_test.py b/tests/cli_test.py index 491479e..0abf36d 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -25,3 +25,15 @@ def test_simple_header_cli(tmp_path: Path, name: str) -> None: res = tf.read_text(encoding="utf-8") assert res == expected + + +def test_parse_failure_sets_exit_code(tmp_path: Path) -> None: + tf = tmp_path / "tmp.h" + result = subprocess.run( + [sys.executable, "-m", "pybind11_mkdoc", "-o", tf, tmp_path / "does_not_exist.h"], + check=False, + capture_output=True, + ) + + assert result.returncode != 0 + assert not tf.exists() diff --git a/tests/sample_header_test.py b/tests/sample_header_test.py index 64f992f..b4aad48 100644 --- a/tests/sample_header_test.py +++ b/tests/sample_header_test.py @@ -23,6 +23,13 @@ def test_generate_headers(capsys, tmp_path): assert output.read_text() == expected +def test_extract_all_twice_in_one_process(): + header = str(DIR / "sample_header_docs" / "sample_header.h") + first = pybind11_mkdoc.mkdoc_lib.extract_all([header]) + second = pybind11_mkdoc.mkdoc_lib.extract_all([header]) + assert first == second + + def test_generate_headers_2(capsys, tmp_path): with open(DIR / "sample_header_docs" / "sample_header_2_truth.h") as f: expected = f.read()