use pytest-xdist to utilize multiple workers for Python tests

.github/workflows/CI-unixish.yml

@@ -262,6 +262,11 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04, ubuntu-22.04, macos-13]
+        include:
+          - xdist_n: auto
+          # FIXME: test_color_tty fails with xdist
+          - os: macos-13
+            xdist_n: '1'
       fail-fast: false # Prefer quick result
 
     runs-on: ${{ matrix.os }}
@@ -300,6 +305,7 @@ jobs:
           python3 -m pip install pip --upgrade
           python3 -m pip install pytest
           python3 -m pip install pytest-timeout
+          python3 -m pip install pytest-xdist
           python3 -m pip install psutil
 
       - name: Build cppcheck
@@ -323,7 +329,7 @@ jobs:
 
       - name: Run test/cli
         run: |
-          python3 -m pytest -Werror --strict-markers -vv test/cli
+          python3 -m pytest -Werror --strict-markers -vv -n ${{ matrix.xdist_n }} test/cli
 
       # TODO: use the step below instead
       # do not use pushd in this step since we go below the working directory
@@ -332,26 +338,26 @@ jobs:
           cd ..
           ln -s cppcheck 'cpp check'
           cd 'cpp check/test/cli'
-          python3 -m pytest -Werror --strict-markers -vv
+          python3 -m pytest -Werror --strict-markers -vv -n ${{ matrix.xdist_n }}
 
       # FIXME: proj2_test.py fails because of the relative path cleanups in ImportProject::setRelativePaths()
       # It fails because the application path used as base path has its symlink resolved by getcwd().
       - name: Run test/cli (symlink)
         if: false
         run: |
           ln -s . 'cpp check'
-          python3 -m pytest -Werror --strict-markers -vv 'cpp check/test/cli'
+          python3 -m pytest -Werror --strict-markers -vv -n ${{ matrix.xdist_n }} 'cpp check/test/cli'
 
       - name: Run test/cli (-j2)
         run: |
-          python3 -m pytest -Werror --strict-markers -vv test/cli
+          python3 -m pytest -Werror --strict-markers -vv -n ${{ matrix.xdist_n }} test/cli
         env:
           TEST_CPPCHECK_INJECT_J: 2
 
       - name: Run test/cli (--clang)
         if: false
         run: |
-          python3 -m pytest -Werror --strict-markers -vv test/cli
+          python3 -m pytest -Werror --strict-markers -vv -n ${{ matrix.xdist_n }} test/cli
         env:
           TEST_CPPCHECK_INJECT_CLANG: clang
 

.github/workflows/CI-windows.yml

@@ -137,6 +137,7 @@ jobs:
           python -m pip install pytest || exit /b !errorlevel!
           python -m pip install pytest-custom_exit_code || exit /b !errorlevel!
           python -m pip install pytest-timeout || exit /b !errorlevel!
+          python -m pip install pytest-xdist || exit /b !errorlevel!
           python -m pip install psutil || exit /b !errorlevel!
 
       # TODO: build with CMake
@@ -176,20 +177,20 @@ jobs:
       - name: Run test/cli
         if: matrix.config == 'release'
         run: |
-          python -m pytest -Werror --strict-markers -vv test/cli || exit /b !errorlevel!
+          python -m pytest -Werror --strict-markers -vv -n auto test/cli || exit /b !errorlevel!
 
       - name: Run test/cli (-j2)
         if: matrix.config == 'release'
         run: |
-          python -m pytest -Werror --strict-markers -vv test/cli || exit /b !errorlevel!
+          python -m pytest -Werror --strict-markers -vv -n auto test/cli || exit /b !errorlevel!
         env:
           TEST_CPPCHECK_INJECT_J: 2
 
       # TODO: install clang
       - name: Run test/cli (--clang)
         if: false # matrix.config == 'release'
         run: |
-          python -m pytest -Werror --strict-markers -vv test/cli || exit /b !errorlevel!
+          python -m pytest -Werror --strict-markers -vv -n auto test/cli || exit /b !errorlevel!
         env:
           TEST_CPPCHECK_INJECT_CLANG: clang
 

.github/workflows/asan.yml

@@ -67,6 +67,7 @@ jobs:
           python3 -m pip install pip --upgrade
           python3 -m pip install pytest
           python3 -m pip install pytest-timeout
+          python3 -m pip install pytest-xdist
           python3 -m pip install psutil
 
       # TODO: disable all warnings
@@ -103,20 +104,20 @@ jobs:
       - name: Run test/cli
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
 
       - name: Run test/cli (-j2)
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_J: 2
 
       - name: Run test/cli (--clang)
         if: false
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_CLANG: clang
 

.github/workflows/scriptcheck.yml

@@ -92,6 +92,7 @@ jobs:
           python -m pip install 'pylint<=3.3.0'
           python -m pip install unittest2
           python -m pip install pytest
+          python -m pip install pytest-xdist
           python -m pip install pygments
           python -m pip install requests
           python -m pip install psutil
@@ -135,18 +136,20 @@ jobs:
         run: |
           python tools/test_matchcompiler.py
 
+      # TODO: run with "-n auto" when misra_test.py can be run in parallel
       # we cannot specify -Werror since xml/etree/ElementTree.py in Python 3.9/3.10 contains an unclosed file
       - name: test addons
         if: matrix.python-version == '3.9' || matrix.python-version == '3.10'
         run: |
-          python -m pytest --strict-markers -vv addons/test
+          python -m pytest --strict-markers -vv -n 1 addons/test
         env:
           PYTHONPATH: ./addons
 
+      # TODO: run with "-n auto" when misra_test.py can be run in parallel
       - name: test addons
         if: matrix.python-version != '3.9' && matrix.python-version != '3.10'
         run: |
-          python -m pytest -Werror --strict-markers -vv addons/test
+          python -m pytest -Werror --strict-markers -vv -n 1 addons/test
         env:
           PYTHONPATH: ./addons
 

.github/workflows/tsan.yml

@@ -67,6 +67,7 @@ jobs:
           python3 -m pip install pip --upgrade
           python3 -m pip install pytest
           python3 -m pip install pytest-timeout
+          python3 -m pip install pytest-xdist
           python3 -m pip install psutil
 
       - name: CMake
@@ -103,22 +104,22 @@ jobs:
       - name: Run test/cli
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_EXECUTOR: thread
 
       - name: Run test/cli (-j2)
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_J: 2
 
       - name: Run test/cli (--clang)
         if: false
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_CLANG: clang
 

.github/workflows/ubsan.yml

@@ -67,6 +67,7 @@ jobs:
           python3 -m pip install pip --upgrade
           python3 -m pip install pytest
           python3 -m pip install pytest-timeout
+          python3 -m pip install pytest-xdist
           python3 -m pip install psutil
 
       # TODO: disable warnings
@@ -103,20 +104,20 @@ jobs:
       - name: Run test/cli
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
 
       - name: Run test/cli (-j2)
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_J: 2
 
       - name: Run test/cli (--clang)
         if: false
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_CLANG: clang
 

addons/test/misra_test.py

@@ -30,6 +30,7 @@ def checker():
     return MisraChecker(settings)
 
 
+# FIXME: files are generates in the source tree so it will cause issues if tests are run with xdist.
 @pytest.fixture
 def test_files():
     for f in TEST_SOURCE_FILES: