From b46a009cebe69742cdf6f70de598cd128cb49b30 Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Mon, 12 Jul 2021 22:41:10 -0700
Subject: [PATCH 01/12] test_iostream.py NUM_RUNS=1000

---
 tests/test_iostream.py | 427 ++++++++++++++++++++++-------------------
 1 file changed, 226 insertions(+), 201 deletions(-)

diff --git a/tests/test_iostream.py b/tests/test_iostream.py
index e2b74d01cb..a0a72957e5 100644
--- a/tests/test_iostream.py
+++ b/tests/test_iostream.py
@@ -40,292 +40,317 @@ def redirect_stderr(target):
         sys.stderr = original
 
 
+NUM_RUNS = 1000
+
+
 def test_captured(capsys):
-    msg = "I've been redirected to Python, I hope!"
-    m.captured_output(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+    for _ in range(NUM_RUNS):
+        msg = "I've been redirected to Python, I hope!"
+        m.captured_output(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
-    m.captured_output_default(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+        m.captured_output_default(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
-    m.captured_err(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == ""
-    assert stderr == msg
+        m.captured_err(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == ""
+        assert stderr == msg
 
 
 def test_captured_large_string(capsys):
-    # Make this bigger than the buffer used on the C++ side: 1024 chars
-    msg = "I've been redirected to Python, I hope!"
-    msg = msg * (1024 // len(msg) + 1)
+    for _ in range(NUM_RUNS):
+        # Make this bigger than the buffer used on the C++ side: 1024 chars
+        msg = "I've been redirected to Python, I hope!"
+        msg = msg * (1024 // len(msg) + 1)
 
-    m.captured_output_default(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+        m.captured_output_default(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
 
 def test_captured_utf8_2byte_offset0(capsys):
-    msg = "\u07FF"
-    msg = "" + msg * (1024 // len(msg) + 1)
+    for _ in range(NUM_RUNS):
+        msg = "\u07FF"
+        msg = "" + msg * (1024 // len(msg) + 1)
 
-    m.captured_output_default(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+        m.captured_output_default(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
 
 def test_captured_utf8_2byte_offset1(capsys):
-    msg = "\u07FF"
-    msg = "1" + msg * (1024 // len(msg) + 1)
+    for _ in range(NUM_RUNS):
+        msg = "\u07FF"
+        msg = "1" + msg * (1024 // len(msg) + 1)
 
-    m.captured_output_default(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+        m.captured_output_default(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
 
 def test_captured_utf8_3byte_offset0(capsys):
-    msg = "\uFFFF"
-    msg = "" + msg * (1024 // len(msg) + 1)
+    for _ in range(NUM_RUNS):
+        msg = "\uFFFF"
+        msg = "" + msg * (1024 // len(msg) + 1)
 
-    m.captured_output_default(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+        m.captured_output_default(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
 
 def test_captured_utf8_3byte_offset1(capsys):
-    msg = "\uFFFF"
-    msg = "1" + msg * (1024 // len(msg) + 1)
+    for _ in range(NUM_RUNS):
+        msg = "\uFFFF"
+        msg = "1" + msg * (1024 // len(msg) + 1)
 
-    m.captured_output_default(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+        m.captured_output_default(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
 
 def test_captured_utf8_3byte_offset2(capsys):
-    msg = "\uFFFF"
-    msg = "12" + msg * (1024 // len(msg) + 1)
+    for _ in range(NUM_RUNS):
+        msg = "\uFFFF"
+        msg = "12" + msg * (1024 // len(msg) + 1)
 
-    m.captured_output_default(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+        m.captured_output_default(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
 
 def test_captured_utf8_4byte_offset0(capsys):
-    msg = "\U0010FFFF"
-    msg = "" + msg * (1024 // len(msg) + 1)
+    for _ in range(NUM_RUNS):
+        msg = "\U0010FFFF"
+        msg = "" + msg * (1024 // len(msg) + 1)
 
-    m.captured_output_default(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+        m.captured_output_default(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
 
 def test_captured_utf8_4byte_offset1(capsys):
-    msg = "\U0010FFFF"
-    msg = "1" + msg * (1024 // len(msg) + 1)
+    for _ in range(NUM_RUNS):
+        msg = "\U0010FFFF"
+        msg = "1" + msg * (1024 // len(msg) + 1)
 
-    m.captured_output_default(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+        m.captured_output_default(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
 
 def test_captured_utf8_4byte_offset2(capsys):
-    msg = "\U0010FFFF"
-    msg = "12" + msg * (1024 // len(msg) + 1)
+    for _ in range(NUM_RUNS):
+        msg = "\U0010FFFF"
+        msg = "12" + msg * (1024 // len(msg) + 1)
 
-    m.captured_output_default(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+        m.captured_output_default(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
 
 def test_captured_utf8_4byte_offset3(capsys):
-    msg = "\U0010FFFF"
-    msg = "123" + msg * (1024 // len(msg) + 1)
+    for _ in range(NUM_RUNS):
+        msg = "\U0010FFFF"
+        msg = "123" + msg * (1024 // len(msg) + 1)
 
-    m.captured_output_default(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+        m.captured_output_default(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
 
 def test_guard_capture(capsys):
-    msg = "I've been redirected to Python, I hope!"
-    m.guard_output(msg)
-    stdout, stderr = capsys.readouterr()
-    assert stdout == msg
-    assert stderr == ""
+    for _ in range(NUM_RUNS):
+        msg = "I've been redirected to Python, I hope!"
+        m.guard_output(msg)
+        stdout, stderr = capsys.readouterr()
+        assert stdout == msg
+        assert stderr == ""
 
 
 def test_series_captured(capture):
-    with capture:
-        m.captured_output("a")
-        m.captured_output("b")
-    assert capture == "ab"
+    for _ in range(NUM_RUNS):
+        with capture:
+            m.captured_output("a")
+            m.captured_output("b")
+        assert capture == "ab"
 
 
 def test_flush(capfd):
-    msg = "(not flushed)"
-    msg2 = "(flushed)"
+    for _ in range(NUM_RUNS):
+        msg = "(not flushed)"
+        msg2 = "(flushed)"
 
-    with m.ostream_redirect():
-        m.noisy_function(msg, flush=False)
-        stdout, stderr = capfd.readouterr()
-        assert stdout == ""
+        with m.ostream_redirect():
+            m.noisy_function(msg, flush=False)
+            stdout, stderr = capfd.readouterr()
+            assert stdout == ""
 
-        m.noisy_function(msg2, flush=True)
-        stdout, stderr = capfd.readouterr()
-        assert stdout == msg + msg2
+            m.noisy_function(msg2, flush=True)
+            stdout, stderr = capfd.readouterr()
+            assert stdout == msg + msg2
 
-        m.noisy_function(msg, flush=False)
+            m.noisy_function(msg, flush=False)
 
-    stdout, stderr = capfd.readouterr()
-    assert stdout == msg
+        stdout, stderr = capfd.readouterr()
+        assert stdout == msg
 
 
 def test_not_captured(capfd):
-    msg = "Something that should not show up in log"
-    stream = StringIO()
-    with redirect_stdout(stream):
-        m.raw_output(msg)
-    stdout, stderr = capfd.readouterr()
-    assert stdout == msg
-    assert stderr == ""
-    assert stream.getvalue() == ""
-
-    stream = StringIO()
-    with redirect_stdout(stream):
-        m.captured_output(msg)
-    stdout, stderr = capfd.readouterr()
-    assert stdout == ""
-    assert stderr == ""
-    assert stream.getvalue() == msg
+    for _ in range(NUM_RUNS):
+        msg = "Something that should not show up in log"
+        stream = StringIO()
+        with redirect_stdout(stream):
+            m.raw_output(msg)
+        stdout, stderr = capfd.readouterr()
+        assert stdout == msg
+        assert stderr == ""
+        assert stream.getvalue() == ""
+
+        stream = StringIO()
+        with redirect_stdout(stream):
+            m.captured_output(msg)
+        stdout, stderr = capfd.readouterr()
+        assert stdout == ""
+        assert stderr == ""
+        assert stream.getvalue() == msg
 
 
 def test_err(capfd):
-    msg = "Something that should not show up in log"
-    stream = StringIO()
-    with redirect_stderr(stream):
-        m.raw_err(msg)
-    stdout, stderr = capfd.readouterr()
-    assert stdout == ""
-    assert stderr == msg
-    assert stream.getvalue() == ""
-
-    stream = StringIO()
-    with redirect_stderr(stream):
-        m.captured_err(msg)
-    stdout, stderr = capfd.readouterr()
-    assert stdout == ""
-    assert stderr == ""
-    assert stream.getvalue() == msg
+    for _ in range(NUM_RUNS):
+        msg = "Something that should not show up in log"
+        stream = StringIO()
+        with redirect_stderr(stream):
+            m.raw_err(msg)
+        stdout, stderr = capfd.readouterr()
+        assert stdout == ""
+        assert stderr == msg
+        assert stream.getvalue() == ""
+
+        stream = StringIO()
+        with redirect_stderr(stream):
+            m.captured_err(msg)
+        stdout, stderr = capfd.readouterr()
+        assert stdout == ""
+        assert stderr == ""
+        assert stream.getvalue() == msg
 
 
 def test_multi_captured(capfd):
-    stream = StringIO()
-    with redirect_stdout(stream):
-        m.captured_output("a")
-        m.raw_output("b")
-        m.captured_output("c")
-        m.raw_output("d")
-    stdout, stderr = capfd.readouterr()
-    assert stdout == "bd"
-    assert stream.getvalue() == "ac"
+    for _ in range(NUM_RUNS):
+        stream = StringIO()
+        with redirect_stdout(stream):
+            m.captured_output("a")
+            m.raw_output("b")
+            m.captured_output("c")
+            m.raw_output("d")
+        stdout, stderr = capfd.readouterr()
+        assert stdout == "bd"
+        assert stream.getvalue() == "ac"
 
 
 def test_dual(capsys):
-    m.captured_dual("a", "b")
-    stdout, stderr = capsys.readouterr()
-    assert stdout == "a"
-    assert stderr == "b"
+    for _ in range(NUM_RUNS):
+        m.captured_dual("a", "b")
+        stdout, stderr = capsys.readouterr()
+        assert stdout == "a"
+        assert stderr == "b"
 
 
 def test_redirect(capfd):
-    msg = "Should not be in log!"
-    stream = StringIO()
-    with redirect_stdout(stream):
-        m.raw_output(msg)
-    stdout, stderr = capfd.readouterr()
-    assert stdout == msg
-    assert stream.getvalue() == ""
-
-    stream = StringIO()
-    with redirect_stdout(stream):
-        with m.ostream_redirect():
+    for _ in range(NUM_RUNS):
+        msg = "Should not be in log!"
+        stream = StringIO()
+        with redirect_stdout(stream):
             m.raw_output(msg)
-    stdout, stderr = capfd.readouterr()
-    assert stdout == ""
-    assert stream.getvalue() == msg
-
-    stream = StringIO()
-    with redirect_stdout(stream):
-        m.raw_output(msg)
-    stdout, stderr = capfd.readouterr()
-    assert stdout == msg
-    assert stream.getvalue() == ""
-
+        stdout, stderr = capfd.readouterr()
+        assert stdout == msg
+        assert stream.getvalue() == ""
 
-def test_redirect_err(capfd):
-    msg = "StdOut"
-    msg2 = "StdErr"
+        stream = StringIO()
+        with redirect_stdout(stream):
+            with m.ostream_redirect():
+                m.raw_output(msg)
+        stdout, stderr = capfd.readouterr()
+        assert stdout == ""
+        assert stream.getvalue() == msg
 
-    stream = StringIO()
-    with redirect_stderr(stream):
-        with m.ostream_redirect(stdout=False):
+        stream = StringIO()
+        with redirect_stdout(stream):
             m.raw_output(msg)
-            m.raw_err(msg2)
-    stdout, stderr = capfd.readouterr()
-    assert stdout == msg
-    assert stderr == ""
-    assert stream.getvalue() == msg2
+        stdout, stderr = capfd.readouterr()
+        assert stdout == msg
+        assert stream.getvalue() == ""
 
 
-def test_redirect_both(capfd):
-    msg = "StdOut"
-    msg2 = "StdErr"
+def test_redirect_err(capfd):
+    for _ in range(NUM_RUNS):
+        msg = "StdOut"
+        msg2 = "StdErr"
 
-    stream = StringIO()
-    stream2 = StringIO()
-    with redirect_stdout(stream):
-        with redirect_stderr(stream2):
-            with m.ostream_redirect():
+        stream = StringIO()
+        with redirect_stderr(stream):
+            with m.ostream_redirect(stdout=False):
                 m.raw_output(msg)
                 m.raw_err(msg2)
-    stdout, stderr = capfd.readouterr()
-    assert stdout == ""
-    assert stderr == ""
-    assert stream.getvalue() == msg
-    assert stream2.getvalue() == msg2
+        stdout, stderr = capfd.readouterr()
+        assert stdout == msg
+        assert stderr == ""
+        assert stream.getvalue() == msg2
+
+
+def test_redirect_both(capfd):
+    for _ in range(NUM_RUNS):
+        msg = "StdOut"
+        msg2 = "StdErr"
+
+        stream = StringIO()
+        stream2 = StringIO()
+        with redirect_stdout(stream):
+            with redirect_stderr(stream2):
+                with m.ostream_redirect():
+                    m.raw_output(msg)
+                    m.raw_err(msg2)
+        stdout, stderr = capfd.readouterr()
+        assert stdout == ""
+        assert stderr == ""
+        assert stream.getvalue() == msg
+        assert stream2.getvalue() == msg2
 
 
 def test_threading():
-    with m.ostream_redirect(stdout=True, stderr=False):
-        # start some threads
-        threads = []
+    for _ in range(NUM_RUNS):
+        with m.ostream_redirect(stdout=True, stderr=False):
+            # start some threads
+            threads = []
 
-        # start some threads
-        for _j in range(20):
-            threads.append(m.TestThread())
+            # start some threads
+            for _j in range(20):
+                threads.append(m.TestThread())
 
-        # give the threads some time to fail
-        threads[0].sleep()
+            # give the threads some time to fail
+            threads[0].sleep()
 
-        # stop all the threads
-        for t in threads:
-            t.stop()
+            # stop all the threads
+            for t in threads:
+                t.stop()
 
-        for t in threads:
-            t.join()
+            for t in threads:
+                t.join()
 
-        # if a thread segfaults, we don't get here
-        assert True
+            # if a thread segfaults, we don't get here
+            assert True

From 42d9a1512b56699b01399d4a6cd33ded1f3c87c7 Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Tue, 13 Jul 2021 09:33:59 -0700
Subject: [PATCH 02/12] valgrind off (it takes many hours to finish)

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 03c9225c12..db8d2af97b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -176,7 +176,7 @@ jobs:
         include:
         - python-version: 3.9
           python-debug: true
-          valgrind: true
+          valgrind: false
         - python-version: 3.10-dev
           python-debug: false
 

From 629810c800859a5889631067692d22873a2c58c6 Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Tue, 13 Jul 2021 09:35:07 -0700
Subject: [PATCH 03/12] NUM_RUNS=1001 (to get more data)

---
 tests/test_iostream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_iostream.py b/tests/test_iostream.py
index a0a72957e5..94f0e6d749 100644
--- a/tests/test_iostream.py
+++ b/tests/test_iostream.py
@@ -40,7 +40,7 @@ def redirect_stderr(target):
         sys.stderr = original
 
 
-NUM_RUNS = 1000
+NUM_RUNS = 1001
 
 
 def test_captured(capsys):

From 4ef4fd7ca7f6f1d70e62c904c662ca6446db2d8f Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Tue, 13 Jul 2021 10:50:22 -0700
Subject: [PATCH 04/12] NUM_RUNS=1002 (to get more data)

---
 tests/test_iostream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_iostream.py b/tests/test_iostream.py
index 94f0e6d749..f221cd94fc 100644
--- a/tests/test_iostream.py
+++ b/tests/test_iostream.py
@@ -40,7 +40,7 @@ def redirect_stderr(target):
         sys.stderr = original
 
 
-NUM_RUNS = 1001
+NUM_RUNS = 1002
 
 
 def test_captured(capsys):

From fab1fce9ced7ac85dd066ab0a4194860fb6697d2 Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Tue, 13 Jul 2021 11:20:10 -0700
Subject: [PATCH 05/12] NUM_RUNS=1003 (to get more data)

---
 tests/test_iostream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_iostream.py b/tests/test_iostream.py
index f221cd94fc..4310b1c973 100644
--- a/tests/test_iostream.py
+++ b/tests/test_iostream.py
@@ -40,7 +40,7 @@ def redirect_stderr(target):
         sys.stderr = original
 
 
-NUM_RUNS = 1002
+NUM_RUNS = 1003
 
 
 def test_captured(capsys):

From 8790297affa3cc27ff21ca736bc2ebf5cad985b1 Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Tue, 13 Jul 2021 16:14:45 -0700
Subject: [PATCH 06/12] Upgrading to pypy-3.7

---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index db8d2af97b..bbec1bed85 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
         - 3.9
         - 3.10-dev
         - pypy2
-        - pypy3
+        - pypy-3.7
 
         # Items in here will either be added to the build matrix (if not
         # present), or add new keys to an existing matrix element if all the
@@ -49,7 +49,7 @@ jobs:
           - runs-on: windows-latest
             python: pypy2
           - runs-on: windows-latest
-            python: pypy3
+            python: pypy-3.7
 
           # TODO: PyPy2 7.3.3 segfaults, while 7.3.2 was fine.
           - runs-on: ubuntu-latest
@@ -716,7 +716,7 @@ jobs:
         - 3.7
         - 3.8
         - 3.9
-        - pypy3
+        - pypy-3.7
         # TODO: fix hang on pypy2
 
         include:

From 3ca03c89228438b87506b11a307968e008bce859 Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Thu, 15 Jul 2021 10:42:17 -0700
Subject: [PATCH 07/12] Reverting all changes relative to current master.

---
 .github/workflows/ci.yml |   8 +-
 tests/test_iostream.py   | 427 ++++++++++++++++++---------------------
 2 files changed, 205 insertions(+), 230 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bbec1bed85..03c9225c12 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
         - 3.9
         - 3.10-dev
         - pypy2
-        - pypy-3.7
+        - pypy3
 
         # Items in here will either be added to the build matrix (if not
         # present), or add new keys to an existing matrix element if all the
@@ -49,7 +49,7 @@ jobs:
           - runs-on: windows-latest
             python: pypy2
           - runs-on: windows-latest
-            python: pypy-3.7
+            python: pypy3
 
           # TODO: PyPy2 7.3.3 segfaults, while 7.3.2 was fine.
           - runs-on: ubuntu-latest
@@ -176,7 +176,7 @@ jobs:
         include:
         - python-version: 3.9
           python-debug: true
-          valgrind: false
+          valgrind: true
         - python-version: 3.10-dev
           python-debug: false
 
@@ -716,7 +716,7 @@ jobs:
         - 3.7
         - 3.8
         - 3.9
-        - pypy-3.7
+        - pypy3
         # TODO: fix hang on pypy2
 
         include:
diff --git a/tests/test_iostream.py b/tests/test_iostream.py
index 4310b1c973..e2b74d01cb 100644
--- a/tests/test_iostream.py
+++ b/tests/test_iostream.py
@@ -40,317 +40,292 @@ def redirect_stderr(target):
         sys.stderr = original
 
 
-NUM_RUNS = 1003
-
-
 def test_captured(capsys):
-    for _ in range(NUM_RUNS):
-        msg = "I've been redirected to Python, I hope!"
-        m.captured_output(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    msg = "I've been redirected to Python, I hope!"
+    m.captured_output(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
-        m.captured_output_default(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    m.captured_output_default(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
-        m.captured_err(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == ""
-        assert stderr == msg
+    m.captured_err(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == ""
+    assert stderr == msg
 
 
 def test_captured_large_string(capsys):
-    for _ in range(NUM_RUNS):
-        # Make this bigger than the buffer used on the C++ side: 1024 chars
-        msg = "I've been redirected to Python, I hope!"
-        msg = msg * (1024 // len(msg) + 1)
+    # Make this bigger than the buffer used on the C++ side: 1024 chars
+    msg = "I've been redirected to Python, I hope!"
+    msg = msg * (1024 // len(msg) + 1)
 
-        m.captured_output_default(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    m.captured_output_default(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
 
 def test_captured_utf8_2byte_offset0(capsys):
-    for _ in range(NUM_RUNS):
-        msg = "\u07FF"
-        msg = "" + msg * (1024 // len(msg) + 1)
+    msg = "\u07FF"
+    msg = "" + msg * (1024 // len(msg) + 1)
 
-        m.captured_output_default(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    m.captured_output_default(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
 
 def test_captured_utf8_2byte_offset1(capsys):
-    for _ in range(NUM_RUNS):
-        msg = "\u07FF"
-        msg = "1" + msg * (1024 // len(msg) + 1)
+    msg = "\u07FF"
+    msg = "1" + msg * (1024 // len(msg) + 1)
 
-        m.captured_output_default(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    m.captured_output_default(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
 
 def test_captured_utf8_3byte_offset0(capsys):
-    for _ in range(NUM_RUNS):
-        msg = "\uFFFF"
-        msg = "" + msg * (1024 // len(msg) + 1)
+    msg = "\uFFFF"
+    msg = "" + msg * (1024 // len(msg) + 1)
 
-        m.captured_output_default(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    m.captured_output_default(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
 
 def test_captured_utf8_3byte_offset1(capsys):
-    for _ in range(NUM_RUNS):
-        msg = "\uFFFF"
-        msg = "1" + msg * (1024 // len(msg) + 1)
+    msg = "\uFFFF"
+    msg = "1" + msg * (1024 // len(msg) + 1)
 
-        m.captured_output_default(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    m.captured_output_default(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
 
 def test_captured_utf8_3byte_offset2(capsys):
-    for _ in range(NUM_RUNS):
-        msg = "\uFFFF"
-        msg = "12" + msg * (1024 // len(msg) + 1)
+    msg = "\uFFFF"
+    msg = "12" + msg * (1024 // len(msg) + 1)
 
-        m.captured_output_default(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    m.captured_output_default(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
 
 def test_captured_utf8_4byte_offset0(capsys):
-    for _ in range(NUM_RUNS):
-        msg = "\U0010FFFF"
-        msg = "" + msg * (1024 // len(msg) + 1)
+    msg = "\U0010FFFF"
+    msg = "" + msg * (1024 // len(msg) + 1)
 
-        m.captured_output_default(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    m.captured_output_default(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
 
 def test_captured_utf8_4byte_offset1(capsys):
-    for _ in range(NUM_RUNS):
-        msg = "\U0010FFFF"
-        msg = "1" + msg * (1024 // len(msg) + 1)
+    msg = "\U0010FFFF"
+    msg = "1" + msg * (1024 // len(msg) + 1)
 
-        m.captured_output_default(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    m.captured_output_default(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
 
 def test_captured_utf8_4byte_offset2(capsys):
-    for _ in range(NUM_RUNS):
-        msg = "\U0010FFFF"
-        msg = "12" + msg * (1024 // len(msg) + 1)
+    msg = "\U0010FFFF"
+    msg = "12" + msg * (1024 // len(msg) + 1)
 
-        m.captured_output_default(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    m.captured_output_default(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
 
 def test_captured_utf8_4byte_offset3(capsys):
-    for _ in range(NUM_RUNS):
-        msg = "\U0010FFFF"
-        msg = "123" + msg * (1024 // len(msg) + 1)
+    msg = "\U0010FFFF"
+    msg = "123" + msg * (1024 // len(msg) + 1)
 
-        m.captured_output_default(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    m.captured_output_default(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
 
 def test_guard_capture(capsys):
-    for _ in range(NUM_RUNS):
-        msg = "I've been redirected to Python, I hope!"
-        m.guard_output(msg)
-        stdout, stderr = capsys.readouterr()
-        assert stdout == msg
-        assert stderr == ""
+    msg = "I've been redirected to Python, I hope!"
+    m.guard_output(msg)
+    stdout, stderr = capsys.readouterr()
+    assert stdout == msg
+    assert stderr == ""
 
 
 def test_series_captured(capture):
-    for _ in range(NUM_RUNS):
-        with capture:
-            m.captured_output("a")
-            m.captured_output("b")
-        assert capture == "ab"
+    with capture:
+        m.captured_output("a")
+        m.captured_output("b")
+    assert capture == "ab"
 
 
 def test_flush(capfd):
-    for _ in range(NUM_RUNS):
-        msg = "(not flushed)"
-        msg2 = "(flushed)"
+    msg = "(not flushed)"
+    msg2 = "(flushed)"
 
-        with m.ostream_redirect():
-            m.noisy_function(msg, flush=False)
-            stdout, stderr = capfd.readouterr()
-            assert stdout == ""
+    with m.ostream_redirect():
+        m.noisy_function(msg, flush=False)
+        stdout, stderr = capfd.readouterr()
+        assert stdout == ""
 
-            m.noisy_function(msg2, flush=True)
-            stdout, stderr = capfd.readouterr()
-            assert stdout == msg + msg2
+        m.noisy_function(msg2, flush=True)
+        stdout, stderr = capfd.readouterr()
+        assert stdout == msg + msg2
 
-            m.noisy_function(msg, flush=False)
+        m.noisy_function(msg, flush=False)
 
-        stdout, stderr = capfd.readouterr()
-        assert stdout == msg
+    stdout, stderr = capfd.readouterr()
+    assert stdout == msg
 
 
 def test_not_captured(capfd):
-    for _ in range(NUM_RUNS):
-        msg = "Something that should not show up in log"
-        stream = StringIO()
-        with redirect_stdout(stream):
-            m.raw_output(msg)
-        stdout, stderr = capfd.readouterr()
-        assert stdout == msg
-        assert stderr == ""
-        assert stream.getvalue() == ""
-
-        stream = StringIO()
-        with redirect_stdout(stream):
-            m.captured_output(msg)
-        stdout, stderr = capfd.readouterr()
-        assert stdout == ""
-        assert stderr == ""
-        assert stream.getvalue() == msg
+    msg = "Something that should not show up in log"
+    stream = StringIO()
+    with redirect_stdout(stream):
+        m.raw_output(msg)
+    stdout, stderr = capfd.readouterr()
+    assert stdout == msg
+    assert stderr == ""
+    assert stream.getvalue() == ""
+
+    stream = StringIO()
+    with redirect_stdout(stream):
+        m.captured_output(msg)
+    stdout, stderr = capfd.readouterr()
+    assert stdout == ""
+    assert stderr == ""
+    assert stream.getvalue() == msg
 
 
 def test_err(capfd):
-    for _ in range(NUM_RUNS):
-        msg = "Something that should not show up in log"
-        stream = StringIO()
-        with redirect_stderr(stream):
-            m.raw_err(msg)
-        stdout, stderr = capfd.readouterr()
-        assert stdout == ""
-        assert stderr == msg
-        assert stream.getvalue() == ""
-
-        stream = StringIO()
-        with redirect_stderr(stream):
-            m.captured_err(msg)
-        stdout, stderr = capfd.readouterr()
-        assert stdout == ""
-        assert stderr == ""
-        assert stream.getvalue() == msg
+    msg = "Something that should not show up in log"
+    stream = StringIO()
+    with redirect_stderr(stream):
+        m.raw_err(msg)
+    stdout, stderr = capfd.readouterr()
+    assert stdout == ""
+    assert stderr == msg
+    assert stream.getvalue() == ""
+
+    stream = StringIO()
+    with redirect_stderr(stream):
+        m.captured_err(msg)
+    stdout, stderr = capfd.readouterr()
+    assert stdout == ""
+    assert stderr == ""
+    assert stream.getvalue() == msg
 
 
 def test_multi_captured(capfd):
-    for _ in range(NUM_RUNS):
-        stream = StringIO()
-        with redirect_stdout(stream):
-            m.captured_output("a")
-            m.raw_output("b")
-            m.captured_output("c")
-            m.raw_output("d")
-        stdout, stderr = capfd.readouterr()
-        assert stdout == "bd"
-        assert stream.getvalue() == "ac"
+    stream = StringIO()
+    with redirect_stdout(stream):
+        m.captured_output("a")
+        m.raw_output("b")
+        m.captured_output("c")
+        m.raw_output("d")
+    stdout, stderr = capfd.readouterr()
+    assert stdout == "bd"
+    assert stream.getvalue() == "ac"
 
 
 def test_dual(capsys):
-    for _ in range(NUM_RUNS):
-        m.captured_dual("a", "b")
-        stdout, stderr = capsys.readouterr()
-        assert stdout == "a"
-        assert stderr == "b"
+    m.captured_dual("a", "b")
+    stdout, stderr = capsys.readouterr()
+    assert stdout == "a"
+    assert stderr == "b"
 
 
 def test_redirect(capfd):
-    for _ in range(NUM_RUNS):
-        msg = "Should not be in log!"
-        stream = StringIO()
-        with redirect_stdout(stream):
+    msg = "Should not be in log!"
+    stream = StringIO()
+    with redirect_stdout(stream):
+        m.raw_output(msg)
+    stdout, stderr = capfd.readouterr()
+    assert stdout == msg
+    assert stream.getvalue() == ""
+
+    stream = StringIO()
+    with redirect_stdout(stream):
+        with m.ostream_redirect():
             m.raw_output(msg)
-        stdout, stderr = capfd.readouterr()
-        assert stdout == msg
-        assert stream.getvalue() == ""
+    stdout, stderr = capfd.readouterr()
+    assert stdout == ""
+    assert stream.getvalue() == msg
 
-        stream = StringIO()
-        with redirect_stdout(stream):
-            with m.ostream_redirect():
-                m.raw_output(msg)
-        stdout, stderr = capfd.readouterr()
-        assert stdout == ""
-        assert stream.getvalue() == msg
-
-        stream = StringIO()
-        with redirect_stdout(stream):
-            m.raw_output(msg)
-        stdout, stderr = capfd.readouterr()
-        assert stdout == msg
-        assert stream.getvalue() == ""
+    stream = StringIO()
+    with redirect_stdout(stream):
+        m.raw_output(msg)
+    stdout, stderr = capfd.readouterr()
+    assert stdout == msg
+    assert stream.getvalue() == ""
 
 
 def test_redirect_err(capfd):
-    for _ in range(NUM_RUNS):
-        msg = "StdOut"
-        msg2 = "StdErr"
+    msg = "StdOut"
+    msg2 = "StdErr"
 
-        stream = StringIO()
-        with redirect_stderr(stream):
-            with m.ostream_redirect(stdout=False):
-                m.raw_output(msg)
-                m.raw_err(msg2)
-        stdout, stderr = capfd.readouterr()
-        assert stdout == msg
-        assert stderr == ""
-        assert stream.getvalue() == msg2
+    stream = StringIO()
+    with redirect_stderr(stream):
+        with m.ostream_redirect(stdout=False):
+            m.raw_output(msg)
+            m.raw_err(msg2)
+    stdout, stderr = capfd.readouterr()
+    assert stdout == msg
+    assert stderr == ""
+    assert stream.getvalue() == msg2
 
 
 def test_redirect_both(capfd):
-    for _ in range(NUM_RUNS):
-        msg = "StdOut"
-        msg2 = "StdErr"
-
-        stream = StringIO()
-        stream2 = StringIO()
-        with redirect_stdout(stream):
-            with redirect_stderr(stream2):
-                with m.ostream_redirect():
-                    m.raw_output(msg)
-                    m.raw_err(msg2)
-        stdout, stderr = capfd.readouterr()
-        assert stdout == ""
-        assert stderr == ""
-        assert stream.getvalue() == msg
-        assert stream2.getvalue() == msg2
+    msg = "StdOut"
+    msg2 = "StdErr"
+
+    stream = StringIO()
+    stream2 = StringIO()
+    with redirect_stdout(stream):
+        with redirect_stderr(stream2):
+            with m.ostream_redirect():
+                m.raw_output(msg)
+                m.raw_err(msg2)
+    stdout, stderr = capfd.readouterr()
+    assert stdout == ""
+    assert stderr == ""
+    assert stream.getvalue() == msg
+    assert stream2.getvalue() == msg2
 
 
 def test_threading():
-    for _ in range(NUM_RUNS):
-        with m.ostream_redirect(stdout=True, stderr=False):
-            # start some threads
-            threads = []
+    with m.ostream_redirect(stdout=True, stderr=False):
+        # start some threads
+        threads = []
 
-            # start some threads
-            for _j in range(20):
-                threads.append(m.TestThread())
+        # start some threads
+        for _j in range(20):
+            threads.append(m.TestThread())
 
-            # give the threads some time to fail
-            threads[0].sleep()
+        # give the threads some time to fail
+        threads[0].sleep()
 
-            # stop all the threads
-            for t in threads:
-                t.stop()
+        # stop all the threads
+        for t in threads:
+            t.stop()
 
-            for t in threads:
-                t.join()
+        for t in threads:
+            t.join()
 
-            # if a thread segfaults, we don't get here
-            assert True
+        # if a thread segfaults, we don't get here
+        assert True

From 102d9bba7993b478f7b2e4b64a89b08283ad371c Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Thu, 15 Jul 2021 11:50:23 -0700
Subject: [PATCH 08/12] Adding tools/repeat_command.py, using in ci.yml

---
 .github/workflows/ci.yml | 25 ++++++++++++++-----------
 tools/repeat_command.py  | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 11 deletions(-)
 create mode 100644 tools/repeat_command.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 03c9225c12..777cc2d32f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -109,7 +109,7 @@ jobs:
       run: cmake --build . -j 2
 
     - name: Python tests C++11
-      run: cmake --build . --target pytest -j 2
+      run: python tools/repeat_command.py 100 cmake --build . --target pytest -j 2
 
     - name: C++11 tests
       # TODO: Figure out how to load the DLL on Python 3.8+
@@ -137,7 +137,7 @@ jobs:
       run: cmake --build build2 -j 2
 
     - name: Python tests
-      run: cmake --build build2 --target pytest
+      run: python tools/repeat_command.py 100 cmake --build build2 --target pytest
 
     - name: C++ tests
       # TODO: Figure out how to load the DLL on Python 3.8+
@@ -176,7 +176,7 @@ jobs:
         include:
         - python-version: 3.9
           python-debug: true
-          valgrind: true
+          valgrind: false
         - python-version: 3.10-dev
           python-debug: false
 
@@ -237,7 +237,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: cmake --build build --target pytest
+      run: python tools/repeat_command.py 100 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -292,7 +292,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -321,7 +321,7 @@ jobs:
       run: cmake --build build -j2 --verbose
 
     - name: Python tests
-      run: cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
 
 
 # TODO: Internal compiler error - report to NVidia
@@ -359,7 +359,7 @@ jobs:
 #      run: cmake --build build -j 2 --verbose
 #
 #    - name: Python tests
-#      run: cmake --build build --target pytest
+#      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
 #
 #    - name: C++ tests
 #      run: cmake --build build --target cpptest
@@ -460,7 +460,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -617,7 +617,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -653,6 +653,9 @@ jobs:
     - name: Copy tests to new directory
       run: cp -a tests /pybind11-tests
 
+    - name: Copy tools/repeat_command.py to /tmp
+      run: cp -a tools/repeat_command.py /tmp
+
     - name: Make a new test directory
       run: mkdir /build-tests
 
@@ -665,7 +668,7 @@ jobs:
       working-directory: /build-tests
 
     - name: Run tests
-      run: make pytest -j 2
+      run: python3 /tmp/repeat_command.py 100 make pytest -j 2
       working-directory: /build-tests
 
 
@@ -761,7 +764,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Run tests
-      run: cmake --build build -t pytest
+      run: python3 tools/repeat_command.py 100 cmake --build build -t pytest
 
   win32-msvc2015:
     name: "🐍 ${{ matrix.python }} • MSVC 2015 • x64"
diff --git a/tools/repeat_command.py b/tools/repeat_command.py
new file mode 100644
index 0000000000..64df5b18ad
--- /dev/null
+++ b/tools/repeat_command.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import subprocess
+import sys
+
+
+def run(args):
+    num_repeats = int(args[0])
+    cmd_and_args = args[1:]
+    assert num_repeats > 0
+    assert cmd_and_args
+    print("REPEAT_COMMAND:CMD_AND_ARGS", cmd_and_args)
+    print()
+    sys.stdout.flush()
+    first_non_zero_retcode = 0
+    for ix in range(num_repeats):
+        print("REPEAT_COMMAND:CALL", ix + 1)
+        sys.stdout.flush()
+        retcode = subprocess.call(cmd_and_args)
+        print("REPEAT_COMMAND:RETCODE", retcode)
+        print()
+        sys.stdout.flush()
+        if retcode and not first_non_zero_retcode:
+            first_non_zero_retcode = retcode
+    print("REPEAT_COMMAND:FIRST_NON_ZERO_RETCODE", first_non_zero_retcode)
+    print()
+    sys.stdout.flush()
+    return first_non_zero_retcode
+
+
+if __name__ == "__main__":
+    sys.exit(run(args=sys.argv[1:]))

From c3e43aad4ceb68cff24aabbf687b0905b5e787b1 Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Thu, 15 Jul 2021 12:54:43 -0700
Subject: [PATCH 09/12] Also reporting REPEAT_COMMAND:RETCODE_COUNTS

---
 tools/repeat_command.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tools/repeat_command.py b/tools/repeat_command.py
index 64df5b18ad..a724e061ba 100644
--- a/tools/repeat_command.py
+++ b/tools/repeat_command.py
@@ -4,6 +4,7 @@
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import subprocess
 import sys
 
@@ -15,7 +16,7 @@ def run(args):
     assert cmd_and_args
     print("REPEAT_COMMAND:CMD_AND_ARGS", cmd_and_args)
     print()
-    sys.stdout.flush()
+    retcode_counts = collections.defaultdict(int)
     first_non_zero_retcode = 0
     for ix in range(num_repeats):
         print("REPEAT_COMMAND:CALL", ix + 1)
@@ -23,9 +24,10 @@ def run(args):
         retcode = subprocess.call(cmd_and_args)
         print("REPEAT_COMMAND:RETCODE", retcode)
         print()
-        sys.stdout.flush()
+        retcode_counts[retcode] += 1
         if retcode and not first_non_zero_retcode:
             first_non_zero_retcode = retcode
+    print("REPEAT_COMMAND:RETCODE_COUNTS", list(sorted(retcode_counts.items())))
     print("REPEAT_COMMAND:FIRST_NON_ZERO_RETCODE", first_non_zero_retcode)
     print()
     sys.stdout.flush()

From 436978178b506fcd2e471f1e31d3a3cc3ff7c62a Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Thu, 15 Jul 2021 17:33:57 -0700
Subject: [PATCH 10/12] Adding SKIP and no-op tricks, repeat 1.

---
 .github/workflows/ci.yml | 22 +++++++++++-----------
 tools/repeat_command.py  | 10 +++++++++-
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 777cc2d32f..3a7dacc7fb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -109,7 +109,7 @@ jobs:
       run: cmake --build . -j 2
 
     - name: Python tests C++11
-      run: python tools/repeat_command.py 100 cmake --build . --target pytest -j 2
+      run: python tools/repeat_command.py 1 cmake --build . --target pytest -j 2
 
     - name: C++11 tests
       # TODO: Figure out how to load the DLL on Python 3.8+
@@ -137,7 +137,7 @@ jobs:
       run: cmake --build build2 -j 2
 
     - name: Python tests
-      run: python tools/repeat_command.py 100 cmake --build build2 --target pytest
+      run: python tools/repeat_command.py 1 cmake --build build2 --target pytest
 
     - name: C++ tests
       # TODO: Figure out how to load the DLL on Python 3.8+
@@ -176,7 +176,7 @@ jobs:
         include:
         - python-version: 3.9
           python-debug: true
-          valgrind: false
+          valgrind: true
         - python-version: 3.10-dev
           python-debug: false
 
@@ -237,7 +237,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: python tools/repeat_command.py 100 cmake --build build --target pytest
+      run: python tools/repeat_command.py 1 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -292,7 +292,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 1 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -321,7 +321,7 @@ jobs:
       run: cmake --build build -j2 --verbose
 
     - name: Python tests
-      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 1 cmake --build build --target pytest
 
 
 # TODO: Internal compiler error - report to NVidia
@@ -359,7 +359,7 @@ jobs:
 #      run: cmake --build build -j 2 --verbose
 #
 #    - name: Python tests
-#      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
+#      run: python3 tools/repeat_command.py 1 cmake --build build --target pytest
 #
 #    - name: C++ tests
 #      run: cmake --build build --target cpptest
@@ -460,7 +460,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 1 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -617,7 +617,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: python3 tools/repeat_command.py 100 cmake --build build --target pytest
+      run: python3 tools/repeat_command.py 1 cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -668,7 +668,7 @@ jobs:
       working-directory: /build-tests
 
     - name: Run tests
-      run: python3 /tmp/repeat_command.py 100 make pytest -j 2
+      run: python3 /tmp/repeat_command.py 1 make pytest -j 2
       working-directory: /build-tests
 
 
@@ -764,7 +764,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Run tests
-      run: python3 tools/repeat_command.py 100 cmake --build build -t pytest
+      run: python3 tools/repeat_command.py 1 cmake --build build -t pytest
 
   win32-msvc2015:
     name: "🐍 ${{ matrix.python }} • MSVC 2015 • x64"
diff --git a/tools/repeat_command.py b/tools/repeat_command.py
index a724e061ba..3934d5d6f5 100644
--- a/tools/repeat_command.py
+++ b/tools/repeat_command.py
@@ -12,8 +12,16 @@
 def run(args):
     num_repeats = int(args[0])
     cmd_and_args = args[1:]
-    assert num_repeats > 0
     assert cmd_and_args
+    if num_repeats < 0:
+        # Shortcut to turn this script into a completely transparent no-op.
+        return subprocess.call(cmd_and_args)
+    if not num_repeats:
+        # Can be used as a simple trick to skip the command entirely.
+        print("REPEAT_COMMAND:SKIP", cmd_and_args)
+        print()
+        sys.stdout.flush()
+        return
     print("REPEAT_COMMAND:CMD_AND_ARGS", cmd_and_args)
     print()
     retcode_counts = collections.defaultdict(int)

From f2dc146f0906a02cdef513c2c7c01cc76493c67f Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Sat, 7 Aug 2021 07:22:11 -0700
Subject: [PATCH 11/12] [ci skip] repeat_command.py now lives here:
 https://github.com/rwgk/rwgk_config/tree/master/bin

---
 tools/repeat_command.py | 46 -----------------------------------------
 1 file changed, 46 deletions(-)
 delete mode 100644 tools/repeat_command.py

diff --git a/tools/repeat_command.py b/tools/repeat_command.py
deleted file mode 100644
index 3934d5d6f5..0000000000
--- a/tools/repeat_command.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import subprocess
-import sys
-
-
-def run(args):
-    num_repeats = int(args[0])
-    cmd_and_args = args[1:]
-    assert cmd_and_args
-    if num_repeats < 0:
-        # Shortcut to turn this script into a completely transparent no-op.
-        return subprocess.call(cmd_and_args)
-    if not num_repeats:
-        # Can be used as a simple trick to skip the command entirely.
-        print("REPEAT_COMMAND:SKIP", cmd_and_args)
-        print()
-        sys.stdout.flush()
-        return
-    print("REPEAT_COMMAND:CMD_AND_ARGS", cmd_and_args)
-    print()
-    retcode_counts = collections.defaultdict(int)
-    first_non_zero_retcode = 0
-    for ix in range(num_repeats):
-        print("REPEAT_COMMAND:CALL", ix + 1)
-        sys.stdout.flush()
-        retcode = subprocess.call(cmd_and_args)
-        print("REPEAT_COMMAND:RETCODE", retcode)
-        print()
-        retcode_counts[retcode] += 1
-        if retcode and not first_non_zero_retcode:
-            first_non_zero_retcode = retcode
-    print("REPEAT_COMMAND:RETCODE_COUNTS", list(sorted(retcode_counts.items())))
-    print("REPEAT_COMMAND:FIRST_NON_ZERO_RETCODE", first_non_zero_retcode)
-    print()
-    sys.stdout.flush()
-    return first_non_zero_retcode
-
-
-if __name__ == "__main__":
-    sys.exit(run(args=sys.argv[1:]))

From f5b726457b88dd130ed1b45ab0a3722a5d8f44f2 Mon Sep 17 00:00:00 2001
From: "Ralf W. Grosse-Kunstleve" <rwgk@google.com>
Date: Sat, 7 Aug 2021 07:36:41 -0700
Subject: [PATCH 12/12] [ci skip] Removing repeat_command.py, consistent step
 name "Python tests".

---
 .github/workflows/ci.yml | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3a7dacc7fb..ccb8fe903e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -109,7 +109,7 @@ jobs:
       run: cmake --build . -j 2
 
     - name: Python tests C++11
-      run: python tools/repeat_command.py 1 cmake --build . --target pytest -j 2
+      run: cmake --build . --target pytest -j 2
 
     - name: C++11 tests
       # TODO: Figure out how to load the DLL on Python 3.8+
@@ -137,7 +137,7 @@ jobs:
       run: cmake --build build2 -j 2
 
     - name: Python tests
-      run: python tools/repeat_command.py 1 cmake --build build2 --target pytest
+      run: cmake --build build2 --target pytest
 
     - name: C++ tests
       # TODO: Figure out how to load the DLL on Python 3.8+
@@ -237,7 +237,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: python tools/repeat_command.py 1 cmake --build build --target pytest
+      run: cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -292,7 +292,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: python3 tools/repeat_command.py 1 cmake --build build --target pytest
+      run: cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -321,7 +321,7 @@ jobs:
       run: cmake --build build -j2 --verbose
 
     - name: Python tests
-      run: python3 tools/repeat_command.py 1 cmake --build build --target pytest
+      run: cmake --build build --target pytest
 
 
 # TODO: Internal compiler error - report to NVidia
@@ -359,7 +359,7 @@ jobs:
 #      run: cmake --build build -j 2 --verbose
 #
 #    - name: Python tests
-#      run: python3 tools/repeat_command.py 1 cmake --build build --target pytest
+#      run: cmake --build build --target pytest
 #
 #    - name: C++ tests
 #      run: cmake --build build --target cpptest
@@ -460,7 +460,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: python3 tools/repeat_command.py 1 cmake --build build --target pytest
+      run: cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -617,7 +617,7 @@ jobs:
       run: cmake --build build -j 2
 
     - name: Python tests
-      run: python3 tools/repeat_command.py 1 cmake --build build --target pytest
+      run: cmake --build build --target pytest
 
     - name: C++ tests
       run: cmake --build build --target cpptest
@@ -653,9 +653,6 @@ jobs:
     - name: Copy tests to new directory
       run: cp -a tests /pybind11-tests
 
-    - name: Copy tools/repeat_command.py to /tmp
-      run: cp -a tools/repeat_command.py /tmp
-
     - name: Make a new test directory
       run: mkdir /build-tests
 
@@ -667,8 +664,8 @@ jobs:
         -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)")
       working-directory: /build-tests
 
-    - name: Run tests
-      run: python3 /tmp/repeat_command.py 1 make pytest -j 2
+    - name: Python tests
+      run: make pytest -j 2
       working-directory: /build-tests
 
 
@@ -763,8 +760,8 @@ jobs:
     - name: Build C++11
       run: cmake --build build -j 2
 
-    - name: Run tests
-      run: python3 tools/repeat_command.py 1 cmake --build build -t pytest
+    - name: Python tests
+      run: cmake --build build -t pytest
 
   win32-msvc2015:
     name: "🐍 ${{ matrix.python }} • MSVC 2015 • x64"