bpo-31227: regrtest reseeds RNG before each test

Lib/test/libregrtest/main.py

@@ -240,9 +240,10 @@ def find_tests(self, tests):
                 print("Couldn't find starting test (%s), using all tests"
                       % self.ns.start, file=sys.stderr)
 
+        if self.ns.random_seed is None:
+            self.ns.random_seed = random.randrange(2 ** 32)
+
         if self.ns.randomize:
-            if self.ns.random_seed is None:
-                self.ns.random_seed = random.randrange(10000000)
             random.seed(self.ns.random_seed)
             random.shuffle(self.selected)
 
@@ -439,8 +440,8 @@ def run_tests(self):
                    or self.tests or self.ns.args)):
             self.display_header()
 
-        if self.ns.randomize:
-            print("Using random seed", self.ns.random_seed)
+        if self.ns.random_seed is not None:
+            print("Random seed: {}".format(self.ns.random_seed))
 
         if self.ns.forever:
             self.tests = self._test_forever(list(self.selected))

Lib/test/libregrtest/runtest.py

@@ -2,6 +2,7 @@
 import importlib
 import io
 import os
+import random
 import sys
 import time
 import traceback
@@ -161,6 +162,7 @@ def runtest_inner(ns, test, display_failure=True):
         with saved_test_environment(test, ns.verbose, ns.quiet, pgo=ns.pgo) as environment:
             start_time = time.time()
             the_module = importlib.import_module(abstest)
+
             # If the test has a test_main, that will run the appropriate
             # tests.  If not, use normal unittest test loading.
             test_runner = getattr(the_module, "test_main", None)
@@ -173,9 +175,17 @@ def test_runner():
                     if loader.errors:
                         raise Exception("errors while loading tests")
                     support.run_unittest(tests)
-            test_runner()
+
+            def final_test():
+                if ns.random_seed is not None:
+                    # bpo-31174: Reseed the RNG before each test file
+                    # to get reproductible results
+                    random.seed(ns.random_seed)
+                test_runner()
+
+            final_test()
             if ns.huntrleaks:
-                refleak = dash_R(the_module, test, test_runner, ns.huntrleaks)
+                refleak = dash_R(the_module, test, final_test, ns.huntrleaks)
             test_time = time.time() - start_time
         post_test_cleanup()
     except support.ResourceDenied as msg:

Lib/test/test_regrtest.py

@@ -457,9 +457,9 @@ def list_regex(line_format, tests):
         self.check_line(output, 'Tests result: %s' % result)
 
     def parse_random_seed(self, output):
-        match = self.regex_search(r'Using random seed ([0-9]+)', output)
+        match = self.regex_search(r'Random seed: ([0-9]+)', output)
         randseed = int(match.group(1))
-        self.assertTrue(0 <= randseed <= 10000000, randseed)
+        self.assertTrue(0 <= randseed < 2 ** 32, randseed)
         return randseed
 
     def run_command(self, args, input=None, exitcode=0, **kw):
@@ -950,6 +950,41 @@ def test_env_changed(self):
         self.check_executed_tests(output, [testname], env_changed=testname,
                                   fail_env_changed=True)
 
+    def check_random_reseed(self, parallel):
+        # bpo-31174: Each test file should be run with the same random seed
+        code = textwrap.dedent("""
+            import random
+            import unittest
+
+            class Tests(unittest.TestCase):
+                def test_random(self):
+                    print("Rand1000: %s" % random.randint(0, 1000))
+        """)
+        testname = self.create_test(code=code)
+
+        tests = [testname] * 3
+        if parallel:
+            output = self.run_tests("-j3", *tests)
+        else:
+            output = self.run_tests(*tests)
+        self.check_executed_tests(output, tests)
+
+        # Get random numbers
+        numbers = (line for line in output.splitlines()
+                   if line.startswith("Rand1000:"))
+        numbers = (line[9:].strip() for line in numbers)
+        numbers = map(int, numbers)
+
+        # All "random" numbers must be the same
+        numbers = set(numbers)
+        self.assertEqual(len(numbers), 1, numbers)
+
+    def test_random_reseed_sequential(self):
+        self.check_random_reseed(False)
+
+    def test_random_reseed_parallel(self):
+        self.check_random_reseed(True)
+
 
 if __name__ == '__main__':
     unittest.main()

Misc/NEWS.d/next/Tests/2017-08-10-14-11-09.bpo-31174.ARF9nb.rst

@@ -0,0 +1,2 @@
+regrtest now reseeds the random RNG before each test file. Use also more
+entropy for the seed: 2**32 (32 bits) rather than 10_000_000 (24 bits).