Use pytest-xdist worksteal where possible

man-group · Dec 8, 2023 · 75ad4ba · 75ad4ba
1 parent 439e666
commit 75ad4ba
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 1 deletion.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -167,12 +167,14 @@ jobs:
         include:
           - python_deps_ids: [""]
             matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
+            pytest_xdist_mode: "--dist worksteal"
           - python3: 6
             python_deps_ids: ["", -compat36]
             matrix_override:
               - ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}}
               - python_deps_id: -compat36
                 python_deps: requirements-compatibility-py36.txt
+            pytest_xdist_mode: ""  # worksteal Not supported on Python 3.6
           - python3: 8
             python_deps_ids: ["", -compat38]
             matrix_override:
@@ -193,6 +195,7 @@ jobs:
       python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
       persistent_storage: ${{inputs.persistent_storage}}
       compile-override: compile-on-ec2
+      pytest_xdist_mode: ${{matrix.pytest_xdist_mode}}
 
   cpp-test-windows:
     needs: [common_config]

diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml
@@ -11,6 +11,7 @@ on:
       python_deps_ids:   {default: '[""]', type: string, description: build-python-wheels test matrix parameter. JSON string.}
       python3:           {default: -1, type: number, description: Python 3 minor version}
       persistent_storage:      {default: "false", type: string, description: Specifies whether the python tests should tests against real storages e.g. AWS S3  }
+      pytest_xdist_mode: {default: "", type: string, description: additional argument to pass for pytest-xdist}
 jobs:
   start_ec2_runner:
     if: inputs.compile-override == 'compile-on-ec2'
@@ -314,6 +315,7 @@ jobs:
           # Use the Mongo created in the service container above to test against
           CI_MONGO_HOST: mongodb
           HYPOTHESIS_PROFILE: ci_${{matrix.os}}
+          PYTEST_XDIST_MODE: ${{inputs.pytest_xdist_mode}}
 
       - name: Collect crash dumps (Windows)
         if: matrix.os == 'windows' && failure()

diff --git a/build_tooling/parallel_test.sh b/build_tooling/parallel_test.sh
@@ -28,7 +28,7 @@ catch=`{ which catchsegv 2>/dev/null || echo ; } | tail -n 1`
 
     export ARCTICDB_RAND_SEED=$RANDOM
 
-    $catch python -m pytest -n $splits -v --log-file="$TEST_OUTPUT_DIR/pytest-logger.$group.log" \
+    $catch python -m pytest -n $splits $PYTEST_XDIST_MODE -v --log-file="$TEST_OUTPUT_DIR/pytest-logger.$group.log" \
         --junitxml="$TEST_OUTPUT_DIR/pytest.$group.xml" \
         --basetemp="$PARALLEL_TEST_ROOT/temp-pytest-output" \
         "$@" 2>&1 | sed -ur "s#^(tests/.*/([^/]+\.py))?#\2#"