Skip to content

Commit

Permalink
gh-109162: Refactor libregrtest.RunTests (#109177)
Browse files Browse the repository at this point in the history 
* Rename dash_R() runtest_refleak(). The function now gets
  huntrleaks and quiet arguments, instead of 'ns' argument.
* Add attributes to Regrtest and RunTests:

  * verbose
  * quiet
  * huntrleaks
  * test_dir

* Add HuntRefleak class.
  • Loading branch information
@vstinner
vstinner committed Sep 9, 2023
1 parent b4131a1 commit e21c89f
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 60 deletions.
57 changes: 32 additions & 25 deletions Lib/test/libregrtest/main.py
View file
Expand Up @@ -12,7 +12,7 @@
from test.libregrtest.cmdline import _parse_args, Namespace
from test.libregrtest.runtest import (
findtests, split_test_packages, run_single_test, abs_module_name,
PROGRESS_MIN_TIME, State, RunTests, TestResult,
PROGRESS_MIN_TIME, State, RunTests, TestResult, HuntRefleak,
FilterTuple, FilterDict, TestList)
from test.libregrtest.setup import setup_tests, setup_test_dir
from test.libregrtest.pgo import setup_pgo_tests
Expand Down Expand Up @@ -92,6 +92,14 @@ def __init__(self, ns: Namespace):
self.pgo_extended: bool = ns.pgo_extended
self.output_on_failure: bool = ns.verbose3
self.timeout: float | None = ns.timeout
self.verbose: bool = ns.verbose
self.quiet: bool = ns.quiet
if ns.huntrleaks:
self.hunt_refleak: HuntRefleak = HuntRefleak(*ns.huntrleaks)
else:
self.hunt_refleak = None
self.test_dir: str | None = ns.testdir
self.junit_filename: str | None = ns.xmlpath

# tests
self.tests = []
Expand Down Expand Up @@ -200,8 +208,7 @@ def log(self, line=''):
print(line, flush=True)

def display_progress(self, test_index, text):
quiet = self.ns.quiet
if quiet:
if self.quiet:
return

# "[ 51/405/1] test_tcl passed"
Expand All @@ -214,7 +221,6 @@ def display_progress(self, test_index, text):
def find_tests(self):
ns = self.ns
single = ns.single
test_dir = ns.testdir

if single:
self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest')
Expand Down Expand Up @@ -250,7 +256,8 @@ def find_tests(self):
exclude_tests.add(arg)
ns.args = []

alltests = findtests(testdir=test_dir, exclude=exclude_tests)
alltests = findtests(testdir=self.test_dir,
exclude=exclude_tests)

if not self.fromfile:
self.selected = self.tests or ns.args
Expand Down Expand Up @@ -298,14 +305,12 @@ def _list_cases(self, suite):
print(test.id())

def list_cases(self):
ns = self.ns
test_dir = ns.testdir
support.verbose = False
support.set_match_tests(self.match_tests, self.ignore_tests)

skipped = []
for test_name in self.selected:
module_name = abs_module_name(test_name, test_dir)
module_name = abs_module_name(test_name, self.test_dir)
try:
suite = unittest.defaultTestLoader.loadTestsFromName(module_name)
self._list_cases(suite)
Expand All @@ -331,7 +336,6 @@ def get_rerun_match(self, rerun_list) -> FilterDict:
def _rerun_failed_tests(self, need_rerun, runtests: RunTests):
# Configure the runner to re-run tests
ns = self.ns
ns.verbose = True
if ns.use_mp is None:
ns.use_mp = 1

Expand All @@ -349,6 +353,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests):
runtests = runtests.copy(
tests=tuple(tests),
rerun=True,
verbose=True,
forever=False,
fail_fast=False,
match_tests_dict=match_tests_dict,
Expand Down Expand Up @@ -379,7 +384,6 @@ def rerun_failed_tests(self, need_rerun, runtests: RunTests):

def display_result(self, runtests):
pgo = runtests.pgo
quiet = self.ns.quiet
print_slow = self.ns.print_slow

# If running the test suite for PGO then no one cares about results.
Expand All @@ -398,7 +402,7 @@ def display_result(self, runtests):
print(count(len(omitted), "test"), "omitted:")
printlist(omitted)

if self.good and not quiet:
if self.good and not self.quiet:
print()
if (not self.bad
and not self.skipped
Expand All @@ -425,12 +429,12 @@ def display_result(self, runtests):
count(len(self.environment_changed), "test")))
printlist(self.environment_changed)

if self.skipped and not quiet:
if self.skipped and not self.quiet:
print()
print(count(len(self.skipped), "test"), "skipped:")
printlist(self.skipped)

if self.resource_denied and not quiet:
if self.resource_denied and not self.quiet:
print()
print(count(len(self.resource_denied), "test"), "skipped (resource denied):")
printlist(self.resource_denied)
Expand Down Expand Up @@ -684,7 +688,7 @@ def display_summary(self):
print(f"Result: {result}")

def save_xml_result(self):
if not self.ns.xmlpath and not self.testsuite_xml:
if not self.junit_filename and not self.testsuite_xml:
return

import xml.etree.ElementTree as ET
Expand All @@ -703,7 +707,7 @@ def save_xml_result(self):
for k, v in totals.items():
root.set(k, str(v))

xmlpath = os.path.join(os_helper.SAVEDCWD, self.ns.xmlpath)
xmlpath = os.path.join(os_helper.SAVEDCWD, self.junit_filename)
with open(xmlpath, 'wb') as f:
for s in ET.tostringlist(root):
f.write(s)
Expand Down Expand Up @@ -785,7 +789,7 @@ def main(self, tests: TestList | None = None):
ns = self.ns
self.tests = tests

if ns.xmlpath:
if self.junit_filename:
support.junit_xml_list = self.testsuite_xml = []

strip_py_suffix(ns.args)
Expand Down Expand Up @@ -844,16 +848,14 @@ def get_exitcode(self):
return exitcode

def action_run_tests(self):
if self.ns.huntrleaks:
warmup, repetitions, _ = self.ns.huntrleaks
if warmup < 3:
msg = ("WARNING: Running tests with --huntrleaks/-R and less than "
"3 warmup repetitions can give false positives!")
print(msg, file=sys.stdout, flush=True)
if self.hunt_refleak and self.hunt_refleak.warmups < 3:
msg = ("WARNING: Running tests with --huntrleaks/-R and "
"less than 3 warmup repetitions can give false positives!")
print(msg, file=sys.stdout, flush=True)

# For a partial run, we do not need to clutter the output.
if (self.want_header
or not(self.pgo or self.ns.quiet or self.ns.single
or not(self.pgo or self.quiet or self.ns.single
or self.tests or self.ns.args)):
self.display_header()

Expand All @@ -869,7 +871,12 @@ def action_run_tests(self):
pgo=self.pgo,
pgo_extended=self.pgo_extended,
output_on_failure=self.output_on_failure,
timeout=self.timeout)
timeout=self.timeout,
verbose=self.verbose,
quiet=self.quiet,
hunt_refleak=self.hunt_refleak,
test_dir=self.test_dir,
junit_filename=self.junit_filename)

setup_tests(runtests, self.ns)

Expand All @@ -892,7 +899,7 @@ def _main(self):
if self.want_wait:
input("Press any key to continue...")

setup_test_dir(self.ns.testdir)
setup_test_dir(self.test_dir)
self.find_tests()

exitcode = 0
Expand Down
23 changes: 14 additions & 9 deletions Lib/test/libregrtest/refleak.py
View file
Expand Up @@ -4,6 +4,7 @@
from inspect import isabstract
from test import support
from test.support import os_helper
from test.libregrtest.runtest import HuntRefleak
from test.libregrtest.utils import clear_caches

try:
Expand All @@ -19,7 +20,9 @@ def _get_dump(cls):
cls._abc_negative_cache, cls._abc_negative_cache_version)


def dash_R(ns, test_name, test_func):
def runtest_refleak(test_name, test_func,
hunt_refleak: HuntRefleak,
quiet: bool):
"""Run a test multiple times, looking for reference leaks.
Returns:
Expand Down Expand Up @@ -62,9 +65,11 @@ def dash_R(ns, test_name, test_func):
def get_pooled_int(value):
return int_pool.setdefault(value, value)

nwarmup, ntracked, fname = ns.huntrleaks
fname = os.path.join(os_helper.SAVEDCWD, fname)
repcount = nwarmup + ntracked
warmups = hunt_refleak.warmups
runs = hunt_refleak.runs
filename = hunt_refleak.filename
filename = os.path.join(os_helper.SAVEDCWD, filename)
repcount = warmups + runs

# Pre-allocate to ensure that the loop doesn't allocate anything new
rep_range = list(range(repcount))
Expand All @@ -78,7 +83,7 @@ def get_pooled_int(value):
# initialize variables to make pyflakes quiet
rc_before = alloc_before = fd_before = interned_before = 0

if not ns.quiet:
if not quiet:
print("beginning", repcount, "repetitions", file=sys.stderr)
print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
flush=True)
Expand All @@ -102,7 +107,7 @@ def get_pooled_int(value):
rc_after = gettotalrefcount() - interned_after * 2
fd_after = fd_count()

if not ns.quiet:
if not quiet:
print('.', end='', file=sys.stderr, flush=True)

rc_deltas[i] = get_pooled_int(rc_after - rc_before)
Expand All @@ -114,7 +119,7 @@ def get_pooled_int(value):
fd_before = fd_after
interned_before = interned_after

if not ns.quiet:
if not quiet:
print(file=sys.stderr)

# These checkers return False on success, True on failure
Expand Down Expand Up @@ -143,12 +148,12 @@ def check_fd_deltas(deltas):
(fd_deltas, 'file descriptors', check_fd_deltas)
]:
# ignore warmup runs
deltas = deltas[nwarmup:]
deltas = deltas[warmups:]
if checker(deltas):
msg = '%s leaked %s %s, sum=%s' % (
test_name, deltas, item_name, sum(deltas))
print(msg, file=sys.stderr, flush=True)
with open(fname, "a", encoding="utf-8") as refrep:
with open(filename, "a", encoding="utf-8") as refrep:
print(msg, file=refrep)
refrep.flush()
failed = True
Expand Down

0 comments on commit e21c89f

Please sign in to comment.