From 15b885255070a6674d084304f34e253914c94f7a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 21 Oct 2023 00:01:20 +0200 Subject: [PATCH] gh-110932: Fix regrtest for SOURCE_DATE_EPOCH If the SOURCE_DATE_EPOCH environment variable is defined, use its value as the random seed. --- Lib/test/libregrtest/main.py | 16 ++-- Lib/test/libregrtest/runtests.py | 2 +- Lib/test/test_regrtest.py | 73 +++++++++++++++---- ...-10-21-00-10-36.gh-issue-110932.jktjJU.rst | 2 + 4 files changed, 72 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-10-21-00-10-36.gh-issue-110932.jktjJU.rst diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index cb60d5af732b43..595f76e4e13981 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -129,14 +129,16 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Randomize self.randomize: bool = ns.randomize - self.random_seed: int | None = ( - ns.random_seed - if ns.random_seed is not None - else random.getrandbits(32) - ) if 'SOURCE_DATE_EPOCH' in os.environ: self.randomize = False - self.random_seed = None + # SOURCE_DATE_EPOCH should be an integer, but use a string to not + # fail if it's not integer. random.seed() accepts a string. + # https://reproducible-builds.org/docs/source-date-epoch/ + self.random_seed: int | str = os.environ['SOURCE_DATE_EPOCH'] + elif ns.random_seed is None: + self.random_seed = random.getrandbits(32) + else: + self.random_seed = ns.random_seed # tests self.first_runtests: RunTests | None = None @@ -441,7 +443,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: or tests or self.cmdline_args)): display_header(self.use_resources, self.python_cmd) - print("Using random seed", self.random_seed) + print("Using random seed:", self.random_seed) runtests = self.create_run_tests(selected) self.first_runtests = runtests diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 4da312db4cb02e..893b311c31297c 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -91,7 +91,7 @@ class RunTests: use_resources: tuple[str, ...] python_cmd: tuple[str, ...] | None randomize: bool - random_seed: int | None + random_seed: int | str json_file: JsonFile | None def copy(self, **override): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e65d9a89ff8a2f..35076eaee923a1 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -143,18 +143,26 @@ def test_header(self): self.assertTrue(ns.header) def test_randomize(self): - for opt in '-r', '--randomize': + for opt in ('-r', '--randomize'): with self.subTest(opt=opt): ns = self.parse_args([opt]) self.assertTrue(ns.randomize) with os_helper.EnvironmentVarGuard() as env: - env['SOURCE_DATE_EPOCH'] = '1' - + # with SOURCE_DATE_EPOCH + env['SOURCE_DATE_EPOCH'] = '1697839080' ns = self.parse_args(['--randomize']) regrtest = main.Regrtest(ns) self.assertFalse(regrtest.randomize) - self.assertIsNone(regrtest.random_seed) + self.assertIsInstance(regrtest.random_seed, str) + self.assertEqual(regrtest.random_seed, '1697839080') + + # without SOURCE_DATE_EPOCH + del env['SOURCE_DATE_EPOCH'] + ns = self.parse_args(['--randomize']) + regrtest = main.Regrtest(ns) + self.assertTrue(regrtest.randomize) + self.assertIsInstance(regrtest.random_seed, int) def test_randseed(self): ns = self.parse_args(['--randseed', '12345']) @@ -388,7 +396,13 @@ def check_ci_mode(self, args, use_resources, rerun=True): # Check Regrtest attributes which are more reliable than Namespace # which has an unclear API - regrtest = main.Regrtest(ns) + with os_helper.EnvironmentVarGuard() as env: + # Ignore SOURCE_DATE_EPOCH env var if it's set + if 'SOURCE_DATE_EPOCH' in env: + del env['SOURCE_DATE_EPOCH'] + + regrtest = main.Regrtest(ns) + self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) self.assertTrue(regrtest.randomize) @@ -661,21 +675,26 @@ def list_regex(line_format, tests): state = f'{state} then {new_state}' self.check_line(output, f'Result: {state}', full=True) - def parse_random_seed(self, output): - match = self.regex_search(r'Using random seed ([0-9]+)', output) - randseed = int(match.group(1)) - self.assertTrue(0 <= randseed, randseed) - return randseed + def parse_random_seed(self, output: str) -> str: + match = self.regex_search(r'Using random seed: (.*)', output) + return match.group(1) def run_command(self, args, input=None, exitcode=0, **kw): if not input: input = '' if 'stderr' not in kw: kw['stderr'] = subprocess.STDOUT + + env = kw.pop('env', None) + if env is None: + env = dict(os.environ) + env.pop('SOURCE_DATE_EPOCH', None) + proc = subprocess.run(args, text=True, input=input, stdout=subprocess.PIPE, + env=env, **kw) if proc.returncode != exitcode: msg = ("Command %s failed with exit code %s, but exit code %s expected!\n" @@ -751,7 +770,9 @@ def setUp(self): self.regrtest_args.append('-n') def check_output(self, output): - self.parse_random_seed(output) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + self.check_executed_tests(output, self.tests, randomize=True, stats=len(self.tests)) @@ -942,7 +963,7 @@ def test_random(self): test_random = int(match.group(1)) # try to reproduce with the random seed - output = self.run_tests('-r', '--randseed=%s' % randseed, test, + output = self.run_tests('-r', f'--randseed={randseed}', test, exitcode=EXITCODE_NO_TESTS_RAN) randseed2 = self.parse_random_seed(output) self.assertEqual(randseed2, randseed) @@ -953,7 +974,33 @@ def test_random(self): # check that random.seed is used by default output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN) - self.assertIsInstance(self.parse_random_seed(output), int) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + + # check SOURCE_DATE_EPOCH (integer) + timestamp = '1697839080' + env = dict(os.environ, SOURCE_DATE_EPOCH=timestamp) + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, timestamp) + self.check_line(output, 'TESTRANDOM: 520') + + # check SOURCE_DATE_EPOCH (string) + env = dict(os.environ, SOURCE_DATE_EPOCH='XYZ') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, 'XYZ') + self.check_line(output, 'TESTRANDOM: 22') + + # check SOURCE_DATE_EPOCH (empty string) + env = dict(os.environ, SOURCE_DATE_EPOCH='') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, '') + self.check_line(output, 'TESTRANDOM: 984') def test_fromfile(self): # test --fromfile diff --git a/Misc/NEWS.d/next/Tests/2023-10-21-00-10-36.gh-issue-110932.jktjJU.rst b/Misc/NEWS.d/next/Tests/2023-10-21-00-10-36.gh-issue-110932.jktjJU.rst new file mode 100644 index 00000000000000..45bb0774a9abe3 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-21-00-10-36.gh-issue-110932.jktjJU.rst @@ -0,0 +1,2 @@ +Fix regrtest if the ``SOURCE_DATE_EPOCH`` environment variable is defined: +use the variable value as the random seed. Patch by Victor Stinner.