Merge pull request #4894 from yut23/avoid-parallel-build-race

setup.py

@@ -11,6 +11,7 @@
     check_for_openmp,
     check_for_pyembree,
     create_build_ext,
+    get_python_include_dirs,
     install_ccompiler,
 )
 
@@ -40,6 +41,8 @@
 CPP14_FLAG = CPP14_CONFIG[_COMPILER]
 CPP11_FLAG = CPP11_CONFIG[_COMPILER]
 
+FIXED_INTERP = "fixed_interpolator"
+
 cythonize_aliases = {
     "LIB_DIR": "yt/utilities/lib/",
     "LIB_DIR_GEOM": ["yt/utilities/lib/", "yt/geometry/"],
@@ -52,7 +55,7 @@
     "EWAH_LIBS": std_libs
     + [os.path.abspath(importlib_resources.files("ewah_bool_utils"))],
     "OMP_ARGS": omp_args,
-    "FIXED_INTERP": "yt/utilities/lib/fixed_interpolator.cpp",
+    "FIXED_INTERP": FIXED_INTERP,
     "ARTIO_SOURCE": sorted(glob.glob("yt/frontends/artio/artio_headers/*.c")),
     "CPP14_FLAG": CPP14_FLAG,
     "CPP11_FLAG": CPP11_FLAG,
@@ -88,8 +91,32 @@ def has_ext_modules(self):
 
 
 if __name__ == "__main__":
+    # Avoid a race condition on fixed_interpolator.o during parallel builds by
+    # building it only once and storing it in a static library.
+    # See https://github.com/yt-project/yt/issues/4278 and
+    # https://github.com/pypa/setuptools/issues/3119#issuecomment-2076922303
+    # for the inspiration for this fix.
+
+    # build_clib doesn't add the Python include dirs (for Python.h) by default,
+    # as opposed to build_ext, so we need to add them manually.
+    clib_include_dirs = get_python_include_dirs()
+
+    # fixed_interpolator.cpp uses Numpy types
+    import numpy
+
+    clib_include_dirs.append(numpy.get_include())
+
+    fixed_interp_lib = (
+        FIXED_INTERP,
+        {
+            "sources": ["yt/utilities/lib/fixed_interpolator.cpp"],
+            "include_dirs": clib_include_dirs,
+        },
+    )
+
     setup(
         cmdclass={"sdist": sdist, "build_ext": build_ext},
         distclass=BinaryDistribution,
+        libraries=[fixed_interp_lib],
         ext_modules=[],  # !!! We override this inside build_ext above
     )

setupext.py

@@ -9,6 +9,7 @@
 import tempfile
 from textwrap import dedent
 from concurrent.futures import ThreadPoolExecutor
+from distutils import sysconfig
 from distutils.ccompiler import CCompiler, new_compiler
 from distutils.sysconfig import customize_compiler
 from subprocess import PIPE, Popen
@@ -368,6 +369,31 @@ def _compile(
     CCompiler.compile = _compile
 
 
+def get_python_include_dirs():
+    """Extracted from distutils.command.build_ext.build_ext.finalize_options(),
+    https://github.com/python/cpython/blob/812245ecce2d8344c3748228047bab456816180a/Lib/distutils/command/build_ext.py#L148-L167
+    """
+    include_dirs = []
+
+    # Make sure Python's include directories (for Python.h, pyconfig.h,
+    # etc.) are in the include search path.
+    py_include = sysconfig.get_python_inc()
+    plat_py_include = sysconfig.get_python_inc(plat_specific=1)
+
+    # If in a virtualenv, add its include directory
+    # Issue 16116
+    if sys.exec_prefix != sys.base_exec_prefix:
+        include_dirs.append(os.path.join(sys.exec_prefix, 'include'))
+
+    # Put the Python "system" include dir at the end, so that
+    # any local include dirs take precedence.
+    include_dirs.extend(py_include.split(os.path.pathsep))
+    if plat_py_include != py_include:
+        include_dirs.extend(plat_py_include.split(os.path.pathsep))
+
+    return include_dirs
+
+
 def create_build_ext(lib_exts, cythonize_aliases):
     class build_ext(_build_ext):
         # subclass setuptools extension builder to avoid importing cython and numpy

yt/utilities/lib/grid_traversal.pyx

@@ -1,6 +1,5 @@
 # distutils: include_dirs = LIB_DIR
 # distutils: libraries = STD_LIBS
-# distutils: sources = FIXED_INTERP
 # distutils: language = c++
 # distutils: extra_compile_args = CPP14_FLAG
 # distutils: extra_link_args = CPP14_FLAG
@@ -20,8 +19,6 @@ from libc.math cimport atan2, cos, fabs, floor, sin, sqrt
 
 from yt.utilities.lib.fp_utils cimport fmin
 
-from .fixed_interpolator cimport *
-
 
 @cython.boundscheck(False)
 @cython.wraparound(False)

yt/utilities/lib/image_samplers.pyx

@@ -1,8 +1,7 @@
 # distutils: include_dirs = LIB_DIR
 # distutils: extra_compile_args = CPP14_FLAG OMP_ARGS
 # distutils: extra_link_args = CPP14_FLAG OMP_ARGS
-# distutils: libraries = STD_LIBS
-# distutils: sources = FIXED_INTERP
+# distutils: libraries = STD_LIBS FIXED_INTERP
 # distutils: language = c++
 """
 Image sampler definitions

yt/utilities/lib/marching_cubes.pyx

@@ -1,6 +1,5 @@
 # distutils: include_dirs = LIB_DIR
-# distutils: libraries = STD_LIBS
-# distutils: sources = FIXED_INTERP
+# distutils: libraries = STD_LIBS FIXED_INTERP
 # distutils: language = c++
 """
 Marching cubes implementation

yt/utilities/lib/partitioned_grid.pyx

@@ -1,6 +1,5 @@
-# distutils: sources = FIXED_INTERP
 # distutils: include_dirs = LIB_DIR
-# distutils: libraries = STD_LIBS
+# distutils: libraries = STD_LIBS FIXED_INTERP
 # distutils: language = c++
 # distutils: extra_compile_args = CPP14_FLAG
 # distutils: extra_link_args = CPP14_FLAG