Adding tools/repeat_command.py, using in ci.yml

Author: rwgk

.github/workflows/ci.yml

@@ -109,7 +109,7 @@ jobs:
       run: cmake --build . -j 2
 
     - name: Python tests C++11
-      run: cmake --build . --target pytest -j 2
+      run: python tools/repeat_command.py 100 cmake --build . --target pytest -j 2
 
     - name: C++11 tests
       # TODO: Figure out how to load the DLL on Python 3.8+
@@ -137,7 +137,7 @@ jobs:
       run: cmake --build build2 -j 2
 
     - name: Python tests
-      run: cmake --build build2 --target pytest
+      run: python tools/repeat_command.py 100 cmake --build build2 --target pytest
 
     - name: C++ tests
       # TODO: Figure out how to load the DLL on Python 3.8+
@@ -176,7 +176,7 @@ jobs:
         include:
         - python-version: 3.9
           python-debug: true
-          valgrind: true
+          valgrind: false
         - python-version: 3.10-dev
           python-debug: false
 
@@ -237,7 +237,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: cmake --build build --target pytest
+      run: python tools/repeat_command.py 100 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -292,7 +292,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -321,7 +321,7 @@ jobs:
       run: cmake --build build -j2 --verbose
 
     - name: Python tests
-      run: cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
 
 
 # TODO: Internal compiler error - report to NVidia
@@ -359,7 +359,7 @@ jobs:
 #      run: cmake --build build -j 2 --verbose
 #
 #    - name: Python tests
-#      run: cmake --build build --target pytest
+#      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
 #
 #    - name: C++ tests
 #      run: cmake --build build --target cpptest
@@ -460,7 +460,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -617,7 +617,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -653,6 +653,9 @@ jobs:
     - name: Copy tests to new directory
       run: cp -a tests /pybind11-tests
 
+    - name: Copy tools/repeat_command.py to /tmp
+      run: cp -a tools/repeat_command.py /tmp
+
     - name: Make a new test directory
       run: mkdir /build-tests
 
@@ -665,7 +668,7 @@ jobs:
       working-directory: /build-tests
 
     - name: Run tests
-      run: make pytest -j 2
+      run: python3 /tmp/repeat_command.py 100 make pytest -j 2
       working-directory: /build-tests
 
 
@@ -761,7 +764,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Run tests
-      run: cmake --build build -t pytest
+      run: python3 tools/repeat_command.py 100 cmake --build build -t pytest
 
   win32-msvc2015:
     name: "🐍 ${{ matrix.python }} • MSVC 2015 • x64"

tools/repeat_command.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import subprocess
+import sys
+
+
+def run(args):
+    num_repeats = int(args[0])
+    cmd_and_args = args[1:]
+    assert num_repeats > 0
+    assert cmd_and_args
+    print("REPEAT_COMMAND:CMD_AND_ARGS", cmd_and_args)
+    print()
+    sys.stdout.flush()
+    first_non_zero_retcode = 0
+    for ix in range(num_repeats):
+        print("REPEAT_COMMAND:CALL", ix + 1)
+        sys.stdout.flush()
+        retcode = subprocess.call(cmd_and_args)
+        print("REPEAT_COMMAND:RETCODE", retcode)
+        print()
+        sys.stdout.flush()
+        if retcode and not first_non_zero_retcode:
+            first_non_zero_retcode = retcode
+    print("REPEAT_COMMAND:FIRST_NON_ZERO_RETCODE", first_non_zero_retcode)
+    print()
+    sys.stdout.flush()
+    return first_non_zero_retcode
+
+
+if __name__ == "__main__":
+    sys.exit(run(args=sys.argv[1:]))