From c1cde1a2edfc6ce1569dda65521c9c9fbe1627b0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Aug 2017 14:02:25 +0200 Subject: [PATCH 1/3] bpo-31174: regrtest reseeds RNG before each test regrtest now reseeds the random RNG before each test file. Use also more entropy for the seed: 2**32 (32 bits) rather than 10_000_000 (24 bits). --- Lib/test/libregrtest/main.py | 9 ++++--- Lib/test/libregrtest/runtest.py | 14 ++++++++-- Lib/test/test_regrtest.py | 26 +++++++++++++++++++ .../2017-08-10-14-11-09.bpo-31174.ARF9nb.rst | 2 ++ 4 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2017-08-10-14-11-09.bpo-31174.ARF9nb.rst diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index bc8155bbff2b478..c05b5e7e6048a2d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -239,9 +239,10 @@ def find_tests(self, tests): print("Couldn't find starting test (%s), using all tests" % self.ns.start, file=sys.stderr) + if self.ns.random_seed is None: + self.ns.random_seed = random.randrange(2 ** 32) + if self.ns.randomize: - if self.ns.random_seed is None: - self.ns.random_seed = random.randrange(10000000) random.seed(self.ns.random_seed) random.shuffle(self.selected) @@ -441,8 +442,8 @@ def run_tests(self): or self.tests or self.ns.args)): self.display_header() - if self.ns.randomize: - print("Using random seed", self.ns.random_seed) + if self.ns.random_seed is not None: + print("Random seed: {}".format(self.ns.random_seed)) if self.ns.forever: self.tests = self._test_forever(list(self.selected)) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index dbd463435c781bd..f4cfcbe04400c64 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -2,6 +2,7 @@ import importlib import io import os +import random import sys import time import traceback @@ -161,6 +162,7 @@ def runtest_inner(ns, test, display_failure=True): with saved_test_environment(test, ns.verbose, ns.quiet, pgo=ns.pgo) as environment: start_time = time.time() the_module = importlib.import_module(abstest) + # If the test has a test_main, that will run the appropriate # tests. If not, use normal unittest test loading. test_runner = getattr(the_module, "test_main", None) @@ -173,9 +175,17 @@ def test_runner(): if loader.errors: raise Exception("errors while loading tests") support.run_unittest(tests) - test_runner() + + def final_test(): + if ns.random_seed is not None: + # bpo-31174: Reseed the RNG before each test file + # to get reproductible results + random.seed(ns.random_seed) + test_runner() + + final_test() if ns.huntrleaks: - refleak = dash_R(the_module, test, test_runner, ns.huntrleaks) + refleak = dash_R(the_module, test, final_test, ns.huntrleaks) test_time = time.time() - start_time post_test_cleanup() except support.ResourceDenied as msg: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index b756839748a158d..52d238ed34ede80 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -950,6 +950,32 @@ def test_env_changed(self): self.check_executed_tests(output, [testname], env_changed=testname, fail_env_changed=True) + def test_random_reseed(self): + # bpo-31174: Each test file should be run with the same random seed + code = textwrap.dedent(""" + import random + import unittest + + class Tests(unittest.TestCase): + def test_random(self): + print("Rand1000: %s" % random.randint(0, 1000)) + """) + testname = self.create_test(code=code) + + tests = [testname] * 3 + output = self.run_tests(*tests) + self.check_executed_tests(output, tests) + + # Get random numbers + numbers = (line for line in output.splitlines() + if line.startswith("Rand1000:")) + numbers = (line[9:].strip() for line in numbers) + numbers = map(int, numbers) + + # All "random" numbers must be the same + numbers = set(numbers) + self.assertEqual(len(numbers), 1, numbers) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2017-08-10-14-11-09.bpo-31174.ARF9nb.rst b/Misc/NEWS.d/next/Tests/2017-08-10-14-11-09.bpo-31174.ARF9nb.rst new file mode 100644 index 000000000000000..794758a04c9b787 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-08-10-14-11-09.bpo-31174.ARF9nb.rst @@ -0,0 +1,2 @@ +regrtest now reseeds the random RNG before each test file. Use also more +entropy for the seed: 2**32 (32 bits) rather than 10_000_000 (24 bits). From 40c13c4eaf153f3931c8a8fdf30e0f6958ed7c22 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Aug 2017 15:27:07 +0200 Subject: [PATCH 2/3] test_regrtest: fix parse_random_seed() --- Lib/test/test_regrtest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 52d238ed34ede80..30b014f7e5d98e7 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -457,9 +457,9 @@ def list_regex(line_format, tests): self.check_line(output, 'Tests result: %s' % result) def parse_random_seed(self, output): - match = self.regex_search(r'Using random seed ([0-9]+)', output) + match = self.regex_search(r'Random seed: ([0-9]+)', output) randseed = int(match.group(1)) - self.assertTrue(0 <= randseed <= 10000000, randseed) + self.assertTrue(0 <= randseed < 2 ** 32, randseed) return randseed def run_command(self, args, input=None, exitcode=0, **kw): From 5121473244d459bb29617a6caee755b82fec77bd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 17 Aug 2017 17:44:09 +0200 Subject: [PATCH 3/3] test_regrtest: test also RNG using -jN --- Lib/test/test_regrtest.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 30b014f7e5d98e7..a99f53268ab9416 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -950,7 +950,7 @@ def test_env_changed(self): self.check_executed_tests(output, [testname], env_changed=testname, fail_env_changed=True) - def test_random_reseed(self): + def check_random_reseed(self, parallel): # bpo-31174: Each test file should be run with the same random seed code = textwrap.dedent(""" import random @@ -963,7 +963,10 @@ def test_random(self): testname = self.create_test(code=code) tests = [testname] * 3 - output = self.run_tests(*tests) + if parallel: + output = self.run_tests("-j3", *tests) + else: + output = self.run_tests(*tests) self.check_executed_tests(output, tests) # Get random numbers @@ -976,6 +979,12 @@ def test_random(self): numbers = set(numbers) self.assertEqual(len(numbers), 1, numbers) + def test_random_reseed_sequential(self): + self.check_random_reseed(False) + + def test_random_reseed_parallel(self): + self.check_random_reseed(True) + if __name__ == '__main__': unittest.main()