test_generator_selection fails sporadically when run with xdist

[Apteryks]

Hello,

Occasionally (once out of 10 builds?) on a 24 cores machine , I get a failure from the test_generator_selection test. Here's the last one i got:

=================================== FAILURES ===================================
___________________________ test_generator_selection ___________________________
[gw16] linux -- Python 3.9.9 /gnu/store/j3cx0yaqdpw0mxizp5bayx93pya44dhn-python-wrapper-3.9.9/bin/python

    def test_generator_selection():
        version = sys.version_info
        env_generator = os.environ.get("CMAKE_GENERATOR")
        this_platform = platform.system().lower()
        get_best_generator = get_platform().get_best_generator
        arch = platform.architecture()[0]
    
        if env_generator:
            assert get_best_generator(env_generator).name == env_generator
    
        if this_platform == "windows":
            # assert that we are running a supported version of python
            py_27_32 = (version.major == 2 and version.minor >= 7) or (version.major == 3 and version.minor <= 2)
    
            py_33_34 = version.major == 3 and (3 <= version.minor <= 4)
    
            py_35 = version.major == 3 and version.minor >= 5
    
            assert len(tuple(filter(bool, (py_27_32, py_33_34, py_35)))) == 1
    
            # Expected Visual Studio version
            if py_27_32:
                vs_generator = "Visual Studio 9 2008"
                vs_version = 9
            elif py_33_34:
                vs_generator = "Visual Studio 10 2010"
                vs_version = 10
            else:
                vs_generator = "Visual Studio 14 2015"
                vs_version = 14
            vs_generator += " Win64" if arch == "64bit" else ""
    
            has_vs_2017 = find_visual_studio(vs_version=VS_YEAR_TO_VERSION["2017"])
            has_vs_2019 = find_visual_studio(vs_version=VS_YEAR_TO_VERSION["2019"])
            has_vs_2022 = find_visual_studio(vs_version=VS_YEAR_TO_VERSION["2022"])
    
            # Apply to VS <= 14 (2015)
            has_vs_ide_vcvars = any(
                [
                    os.path.exists(path_pattern % vs_version)
                    for path_pattern in ["C:/Program Files (x86)/Microsoft Visual Studio %.1f/VC/vcvarsall.bat"]
                ]
            )
    
            # As of Dec 2016, this is available only for VS 9.0
            has_vs_for_python_vcvars = any(
                [
                    os.path.exists(os.path.expanduser(path_pattern % vs_version))
                    for path_pattern in [
                        "~/AppData/Local/Programs/Common/Microsoft/Visual C++ for Python/%.1f/vcvarsall.bat",
                        "C:/Program Files (x86)/Common Files/Microsoft/Visual C++ for Python/%.1f/vcvarsall.bat",
                    ]
                ]
            )
    
            # If environment exists, update the expected generator
            if (has_vs_for_python_vcvars or has_vs_ide_vcvars) and which("ninja.exe"):
                assert get_best_generator().name == "Ninja"
    
            elif has_vs_2017:
                vs_generator = "Visual Studio 15 2017"
                # Early versions of 2017 may not ship with Ninja (TODO: check)
                assert get_best_generator().name in {"Ninja", vs_generator}
    
            elif has_vs_2019 or has_vs_2022:
                # ninja is provided by the CMake extension bundled with Visual Studio 2017
                # C:/Program Files (x86)/Microsoft Visual Studio/2017/Professional/Common7/IDE/CommonExtensions/Microsoft/CMake/Ninja/ninja.exe  # noqa: E501
                assert get_best_generator().name == "Ninja"
    
            elif has_vs_ide_vcvars:
                assert get_best_generator().name == vs_generator
    
            elif has_vs_for_python_vcvars:
                assert get_best_generator().name == "NMake Makefiles"
    
        elif this_platform in ["darwin", "linux"]:
            generator = "Ninja" if which("ninja") else "Unix Makefiles"
>           assert get_best_generator().name == generator

arch       = '64bit'
env_generator = None
generator  = 'Ninja'
get_best_generator = <bound method CMakePlatform.get_best_generator of <skbuild.platform_specifics.linux.LinuxPlatform object at 0x7ffff4336dc0>>
this_platform = 'linux'
version    = sys.version_info(major=3, minor=9, micro=9, releaselevel='final', serial=0)

tests/test_skbuild.py:101: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
skbuild/platform_specifics/abstract.py:165: in get_best_generator
    working_generator = self.compile_test_cmakelist(cmake_executable, candidate_generators, cmake_args)
        architecture = None
        candidate_generators = [<skbuild.platform_specifics.abstract.CMakeGenerator object at 0x7ffff4336d60>,
 <skbuild.platform_specifics.abstract.CMakeGenerator object at 0x7ffff4336130>]
        cleanup    = True
        cmake_args = ()
        cmake_executable = '/gnu/store/zga679c4nldah9l8dhd5a4hdy820hcyf-cmake-minimal-3.21.4/bin/cmake'
        generator_name = None
        languages  = ('CXX', 'C')
        self       = <skbuild.platform_specifics.linux.LinuxPlatform object at 0x7ffff4336dc0>
        skip_generator_test = False
skbuild/utils/__init__.py:58: in inner
    return func(*args, **kwds)
        args       = ('/gnu/store/zga679c4nldah9l8dhd5a4hdy820hcyf-cmake-minimal-3.21.4/bin/cmake',
 [<skbuild.platform_specifics.abstract.CMakeGenerator object at 0x7ffff4336d60>,
  <skbuild.platform_specifics.abstract.CMakeGenerator object at 0x7ffff4336130>],
 ())
        func       = <function CMakePlatform.compile_test_cmakelist at 0x7ffff4bbc310>
        kwds       = {}
        self       = <skbuild.utils.push_dir object at 0x7ffff4bba790>
skbuild/platform_specifics/abstract.py:221: in compile_test_cmakelist
    shutil.rmtree("build")
        _generator_discovery_status_msg = <function CMakePlatform.compile_test_cmakelist.<locals>._generator_discovery_status_msg at 0x7ffff430cc10>
        candidate_generators = [<skbuild.platform_specifics.abstract.CMakeGenerator object at 0x7ffff4336d60>,
 <skbuild.platform_specifics.abstract.CMakeGenerator object at 0x7ffff4336130>]
        cmake_args = ['--no-warn-unused-cli']
        cmake_exe_path = '/gnu/store/zga679c4nldah9l8dhd5a4hdy820hcyf-cmake-minimal-3.21.4/bin/cmake'
        generator  = <skbuild.platform_specifics.abstract.CMakeGenerator object at 0x7ffff4336d60>
        working_generator = None
/gnu/store/b6j1qw1a5rkbfvcy7lc9fm95abbzpa4x-python-3.9.9/lib/python3.9/shutil.py:727: in rmtree
    _rmtree_safe_fd(fd, path, onerror)
        fd         = 13
        ignore_errors = False
        onerror    = <function rmtree.<locals>.onerror at 0x7ffff42d5ca0>
        orig_st    = os.stat_result(st_mode=16877, st_ino=911108156, st_dev=25, st_nlink=1, st_uid=999, st_gid=30000, st_size=20, st_atime=1652373044, st_mtime=1652373044, st_ctime=1652373044)
        path       = 'build'
/gnu/store/b6j1qw1a5rkbfvcy7lc9fm95abbzpa4x-python-3.9.9/lib/python3.9/shutil.py:664: in _rmtree_safe_fd
    _rmtree_safe_fd(dirfd, fullname, onerror)
        dirfd      = 14
        entries    = [<DirEntry 'CMakeFiles'>]
        entry      = <DirEntry 'CMakeFiles'>
        fullname   = 'build/CMakeFiles'
        is_dir     = True
        onerror    = <function rmtree.<locals>.onerror at 0x7ffff42d5ca0>
        orig_st    = os.stat_result(st_mode=16877, st_ino=911108176, st_dev=25, st_nlink=1, st_uid=999, st_gid=30000, st_size=58, st_atime=1652373044, st_mtime=1652373046, st_ctime=1652373046)
        path       = 'build'
        scandir_it = <posix.ScandirIterator object at 0x7ffff4328ab0>
        topfd      = 13
/gnu/store/b6j1qw1a5rkbfvcy7lc9fm95abbzpa4x-python-3.9.9/lib/python3.9/shutil.py:664: in _rmtree_safe_fd
    _rmtree_safe_fd(dirfd, fullname, onerror)
        dirfd      = 15
        entries    = [<DirEntry 'CMakeOutput.log'>, <DirEntry '3.21.4'>, <DirEntry 'CMakeTmp'>]
        entry      = <DirEntry 'CMakeTmp'>
        fullname   = 'build/CMakeFiles/CMakeTmp'
        is_dir     = True
        onerror    = <function rmtree.<locals>.onerror at 0x7ffff42d5ca0>
        orig_st    = os.stat_result(st_mode=16877, st_ino=911108689, st_dev=25, st_nlink=1, st_uid=999, st_gid=30000, st_size=148, st_atime=1652373048, st_mtime=1652373048, st_ctime=1652373048)
        path       = 'build/CMakeFiles'
        scandir_it = <posix.ScandirIterator object at 0x7ffff4328a40>
        topfd      = 14
/gnu/store/b6j1qw1a5rkbfvcy7lc9fm95abbzpa4x-python-3.9.9/lib/python3.9/shutil.py:664: in _rmtree_safe_fd
    _rmtree_safe_fd(dirfd, fullname, onerror)
        dirfd      = 16
        entries    = [<DirEntry 'CMakeFiles'>,
 <DirEntry 'CMakeCache.txt'>,
 <DirEntry 'cmake_install.cmake'>,
 <DirEntry 'build.ninja'>,
 <DirEntry '.ninja_log'>,
 <DirEntry 'cmTC_9455c'>]
        entry      = <DirEntry 'CMakeFiles'>
        fullname   = 'build/CMakeFiles/CMakeTmp/CMakeFiles'
        is_dir     = True
        onerror    = <function rmtree.<locals>.onerror at 0x7ffff42d5ca0>
        orig_st    = os.stat_result(st_mode=16877, st_ino=911108749, st_dev=25, st_nlink=1, st_uid=999, st_gid=30000, st_size=92, st_atime=1652373048, st_mtime=1652373048, st_ctime=1652373048)
        path       = 'build/CMakeFiles/CMakeTmp'
        scandir_it = <posix.ScandirIterator object at 0x7ffff42cf110>
        topfd      = 15
/gnu/store/b6j1qw1a5rkbfvcy7lc9fm95abbzpa4x-python-3.9.9/lib/python3.9/shutil.py:664: in _rmtree_safe_fd
    _rmtree_safe_fd(dirfd, fullname, onerror)
        dirfd      = 17
        entries    = [<DirEntry 'cmTC_9455c.dir'>,
 <DirEntry 'TargetDirectories.txt'>,
 <DirEntry 'rules.ninja'>]
        entry      = <DirEntry 'cmTC_9455c.dir'>
        fullname   = 'build/CMakeFiles/CMakeTmp/CMakeFiles/cmTC_9455c.dir'
        is_dir     = True
        onerror    = <function rmtree.<locals>.onerror at 0x7ffff42d5ca0>
        orig_st    = os.stat_result(st_mode=16877, st_ino=911108765, st_dev=25, st_nlink=1, st_uid=999, st_gid=30000, st_size=342, st_atime=1652373046, st_mtime=1652373047, st_ctime=1652373047)
        path       = 'build/CMakeFiles/CMakeTmp/CMakeFiles'
        scandir_it = <posix.ScandirIterator object at 0x7ffff42cf1f0>
        topfd      = 16
/gnu/store/b6j1qw1a5rkbfvcy7lc9fm95abbzpa4x-python-3.9.9/lib/python3.9/shutil.py:684: in _rmtree_safe_fd
    onerror(os.unlink, fullname, sys.exc_info())
        entries    = [<DirEntry 'FortranDependInfo.json'>,
 <DirEntry 'CMakeFortranCompilerABI.F-pp.f'>,
 <DirEntry 'CMakeFortranCompilerABI.F-pp.f.d'>,
 <DirEntry 'CMakeFortranCompilerABI.F.o.ddi'>,
 <DirEntry 'FortranModules.json'>,
 <DirEntry 'Fortran.dd'>,
 <DirEntry 'CMakeFortranCompilerABI.F.o'>]
        entry      = <DirEntry 'CMakeFortranCompilerABI.F.o'>
        fullname   = 'build/CMakeFiles/CMakeTmp/CMakeFiles/cmTC_9455c.dir/CMakeFortranCompilerABI.F.o'
        is_dir     = False
        onerror    = <function rmtree.<locals>.onerror at 0x7ffff42d5ca0>
        path       = 'build/CMakeFiles/CMakeTmp/CMakeFiles/cmTC_9455c.dir'
        scandir_it = <posix.ScandirIterator object at 0x7ffff42cf0a0>
        topfd      = 17
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

topfd = 17, path = 'build/CMakeFiles/CMakeTmp/CMakeFiles/cmTC_9455c.dir'
onerror = <function rmtree.<locals>.onerror at 0x7ffff42d5ca0>

    def _rmtree_safe_fd(topfd, path, onerror):
        try:
            with os.scandir(topfd) as scandir_it:
                entries = list(scandir_it)
        except OSError as err:
            err.filename = path
            onerror(os.scandir, path, sys.exc_info())
            return
        for entry in entries:
            fullname = os.path.join(path, entry.name)
            try:
                is_dir = entry.is_dir(follow_symlinks=False)
            except OSError:
                is_dir = False
            else:
                if is_dir:
                    try:
                        orig_st = entry.stat(follow_symlinks=False)
                        is_dir = stat.S_ISDIR(orig_st.st_mode)
                    except OSError:
                        onerror(os.lstat, fullname, sys.exc_info())
                        continue
            if is_dir:
                try:
                    dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd)
                except OSError:
                    onerror(os.open, fullname, sys.exc_info())
                else:
                    try:
                        if os.path.samestat(orig_st, os.fstat(dirfd)):
                            _rmtree_safe_fd(dirfd, fullname, onerror)
                            try:
                                os.rmdir(entry.name, dir_fd=topfd)
                            except OSError:
                                onerror(os.rmdir, fullname, sys.exc_info())
                        else:
                            try:
                                # This can only happen if someone replaces
                                # a directory with a symlink after the call to
                                # os.scandir or stat.S_ISDIR above.
                                raise OSError("Cannot call rmtree on a symbolic "
                                              "link")
                            except OSError:
                                onerror(os.path.islink, fullname, sys.exc_info())
                    finally:
                        os.close(dirfd)
            else:
                try:
>                   os.unlink(entry.name, dir_fd=topfd)
E                   FileNotFoundError: [Errno 2] No such file or directory: 'CMakeFortranCompilerABI.F.o'

entries    = [<DirEntry 'FortranDependInfo.json'>,
 <DirEntry 'CMakeFortranCompilerABI.F-pp.f'>,
 <DirEntry 'CMakeFortranCompilerABI.F-pp.f.d'>,
 <DirEntry 'CMakeFortranCompilerABI.F.o.ddi'>,
 <DirEntry 'FortranModules.json'>,
 <DirEntry 'Fortran.dd'>,
 <DirEntry 'CMakeFortranCompilerABI.F.o'>]
entry      = <DirEntry 'CMakeFortranCompilerABI.F.o'>
fullname   = 'build/CMakeFiles/CMakeTmp/CMakeFiles/cmTC_9455c.dir/CMakeFortranCompilerABI.F.o'
is_dir     = False
onerror    = <function rmtree.<locals>.onerror at 0x7ffff42d5ca0>
path       = 'build/CMakeFiles/CMakeTmp/CMakeFiles/cmTC_9455c.dir'
scandir_it = <posix.ScandirIterator object at 0x7ffff42cf0a0>
topfd      = 17

/gnu/store/b6j1qw1a5rkbfvcy7lc9fm95abbzpa4x-python-3.9.9/lib/python3.9/shutil.py:682: FileNotFoundError
----------------------------- Captured stdout call -----------------------------


--------------------------------------------------------------------------------
-- Trying "Ninja" generator
--------------------------------
---------------------------
----------------------
-----------------
------------
-------
--
=============================== warnings summary ===============================
tests/test_hello_cython.py::test_hello_cython_sdist
tests/test_hello_cython.py::test_hello_cython_builds
tests/test_hello_cython.py::test_hello_cython_wheel
  setup.py:3: FutureWarning: package_dir={'hello_cython': 'hello/'} ends with a trailing slash, which is not supported by setuptools.

tests/test_hello_cython_manifest.py::test_hello_cython_sdist
tests/test_hello_cython_manifest.py::test_hello_cython_builds
tests/test_hello_cython_manifest.py::test_hello_cython_wheel
  setup.py:3: FutureWarning: package_dir={'hello_cython_manifest': 'hello/'} ends with a trailing slash, which is not supported by setuptools.

-- Docs: https://docs.pytest.org/en/stable/warnings.html

----------- coverage: platform linux, python 3.9.9-final-0 -----------
Coverage XML written to file tests/coverage.xml

=========================== short test summary info ============================
SKIPPED [1] tests/test_logging.py:5: could not import 'setuptools.logging': No module named 'setuptools.logging'
SKIPPED [1] tests/test_setup.py:323: pypi.org website not reachable
SKIPPED [24] tests/test_setup.py:588: unsupported configuration: python package fully generated by CMake does *NOT* work. At least __init__.py should be in the project source tree
SKIPPED [8] tests/test_skbuild.py:156: Requires Windows
SKIPPED [1] tests/test_skbuild.py:178: Requires Windows
SKIPPED [1] tests/test_skbuild.py:112: NMake Makefiles generator is available only on Linux
SKIPPED [1] tests/test_platform.py:117: Requires Windows
FAILED tests/test_skbuild.py::test_generator_selection - FileNotFoundError: [...
============ 1 failed, 191 passed, 37 skipped, 6 warnings in 50.98s ============

This usually means the test reuse the same directory/files across parameterized tests, which cause races and undefined behavior.

[Apteryks]

I also had disabled test_generator_cleanup due to unexplained failures, perhaps it is similarly racy.