[3.7] bpo-36719: sync regrtest with master branch (GH-12967)

* Clean up code which checked presence of os.{stat,lstat,chmod} (GH-11643)

(cherry picked from commit 8377cd4)

* bpo-36725: regrtest: add TestResult type (GH-12960)

* Add TestResult and MultiprocessResult types to ensure that results
  always have the same fields.
* runtest() now handles KeyboardInterrupt
* accumulate_result() and format_test_result() now takes a TestResult
* cleanup_test_droppings() is now called by runtest() and mark the
  test as ENV_CHANGED if the test leaks support.TESTFN file.
* runtest() now includes code "around" the test in the test timing
* Add print_warning() in test.libregrtest.utils to standardize how
  libregrtest logs warnings to ease parsing the test output.
* support.unload() is now called with abstest rather than test_name
* Rename 'test' variable/parameter to 'test_name'
* dash_R(): remove unused the_module parameter
* Remove unused imports

(cherry picked from commit 4d29983)

* bpo-36725: Refactor regrtest multiprocessing code (GH-12961)

Rewrite run_tests_multiprocess() function as a new MultiprocessRunner
class with multiple methods to better report errors and stop
immediately when needed.

Changes:

* Worker processes are now killed immediately if tests are
  interrupted or if a test does crash (CHILD_ERROR): worker
  processes are killed.
* Rewrite how errors in a worker thread are reported to
  the main thread. No longer ignore BaseException or parsing errors
  silently.
* Remove 'finished' variable: use worker.is_alive() instead
* Always compute omitted tests. Add Regrtest.get_executed() method.

(cherry picked from commit 3cde440)

* bpo-36719: regrtest always detect uncollectable objects (GH-12951)

regrtest now always detects uncollectable objects. Previously, the
check was only enabled by --findleaks. The check now also works with
-jN/--multiprocess N.

--findleaks becomes a deprecated alias to --fail-env-changed.

(cherry picked from commit 75120d2)

* bpo-34060: Report system load when running test suite for Windows (GH-8357)

While Windows exposes the system processor queue length, the raw value
used for load calculations on Unix systems, it does not provide an API
to access the averaged value. Hence to calculate the load we must track
and average it ourselves. We can't use multiprocessing or a thread to
read it in the background while the tests run since using those would
conflict with test_multiprocessing and test_xxsubprocess.

Thus, we use Window's asynchronous IO API to run the tracker in the
background with it sampling at the correct rate. When we wish to access
the load we check to see if there's new data on the stream, if there is,
we update our load values.


(cherry picked from commit e16467a)

* bpo-36719: Fix regrtest re-run (GH-12964)

Properly handle a test which fail but then pass.

Add test_rerun_success() unit test.

(cherry picked from commit 837acc1)

* bpo-36719: regrtest closes explicitly WindowsLoadTracker (GH-12965)

Regrtest.finalize() now closes explicitly the WindowsLoadTracker
instance.

(cherry picked from commit 00db7c7)

Lib/test/libregrtest/cmdline.py

@@ -226,8 +226,9 @@ def _create_parser():
                             '(instead of the Python stdlib test suite)')
 
     group = parser.add_argument_group('Special runs')
-    group.add_argument('-l', '--findleaks', action='store_true',
-                       help='if GC is available detect tests that leak memory')
+    group.add_argument('-l', '--findleaks', action='store_const', const=2,
+                       default=1,
+                       help='deprecated alias to --fail-env-changed')
     group.add_argument('-L', '--runleaks', action='store_true',
                        help='run the leaks(1) command just before exit.' +
                             more_details)
@@ -309,7 +310,7 @@ def _parse_args(args, **kwargs):
     # Defaults
     ns = argparse.Namespace(testdir=None, verbose=0, quiet=False,
          exclude=False, single=False, randomize=False, fromfile=None,
-         findleaks=False, use_resources=None, trace=False, coverdir='coverage',
+         findleaks=1, use_resources=None, trace=False, coverdir='coverage',
          runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
          random_seed=None, use_mp=None, verbose3=False, forever=False,
          header=False, failfast=False, match_tests=None, pgo=False)
@@ -330,12 +331,13 @@ def _parse_args(args, **kwargs):
             parser.error("unrecognized arguments: %s" % arg)
             sys.exit(1)
 
+    if ns.findleaks > 1:
+        # --findleaks implies --fail-env-changed
+        ns.fail_env_changed = True
     if ns.single and ns.fromfile:
         parser.error("-s and -f don't go together!")
     if ns.use_mp is not None and ns.trace:
         parser.error("-T and -j don't go together!")
-    if ns.use_mp is not None and ns.findleaks:
-        parser.error("-l and -j don't go together!")
     if ns.failfast and not (ns.verbose or ns.verbose3):
         parser.error("-G/--failfast needs either -v or -W")
     if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3):

Lib/test/libregrtest/main.py

@@ -20,10 +20,6 @@
 from test.libregrtest.setup import setup_tests
 from test.libregrtest.utils import removepy, count, format_duration, printlist
 from test import support
-try:
-    import gc
-except ImportError:
-    gc = None
 
 
 # When tests are run from the Python build directory, it is best practice
@@ -79,8 +75,8 @@ def __init__(self):
         self.skipped = []
         self.resource_denieds = []
         self.environment_changed = []
-        self.rerun = []
         self.run_no_tests = []
+        self.rerun = []
         self.first_result = None
         self.interrupted = False
 
@@ -90,9 +86,6 @@ def __init__(self):
         # used by --coverage, trace.Trace instance
         self.tracer = None
 
-        # used by --findleaks, store for gc.garbage
-        self.found_garbage = []
-
         # used to display the progress bar "[ 3/100]"
         self.start_time = time.monotonic()
         self.test_count = ''
@@ -105,26 +98,43 @@ def __init__(self):
         # used by --junit-xml
         self.testsuite_xml = None
 
-    def accumulate_result(self, test, result):
-        ok, test_time, xml_data = result
-        if ok not in (CHILD_ERROR, INTERRUPTED):
-            self.test_times.append((test_time, test))
+        self.win_load_tracker = None
+
+    def get_executed(self):
+        return (set(self.good) | set(self.bad) | set(self.skipped)
+                | set(self.resource_denieds) | set(self.environment_changed)
+                | set(self.run_no_tests))
+
+    def accumulate_result(self, result, rerun=False):
+        test_name = result.test_name
+        ok = result.result
+
+        if ok not in (CHILD_ERROR, INTERRUPTED) and not rerun:
+            self.test_times.append((result.test_time, test_name))
+
         if ok == PASSED:
-            self.good.append(test)
+            self.good.append(test_name)
         elif ok in (FAILED, CHILD_ERROR):
-            self.bad.append(test)
+            if not rerun:
+                self.bad.append(test_name)
         elif ok == ENV_CHANGED:
-            self.environment_changed.append(test)
+            self.environment_changed.append(test_name)
         elif ok == SKIPPED:
-            self.skipped.append(test)
+            self.skipped.append(test_name)
         elif ok == RESOURCE_DENIED:
-            self.skipped.append(test)
-            self.resource_denieds.append(test)
+            self.skipped.append(test_name)
+            self.resource_denieds.append(test_name)
         elif ok == TEST_DID_NOT_RUN:
-            self.run_no_tests.append(test)
-        elif ok != INTERRUPTED:
+            self.run_no_tests.append(test_name)
+        elif ok == INTERRUPTED:
+            self.interrupted = True
+        else:
             raise ValueError("invalid test result: %r" % ok)
 
+        if rerun and ok not in {FAILED, CHILD_ERROR, INTERRUPTED}:
+            self.bad.remove(test_name)
+
+        xml_data = result.xml_data
         if xml_data:
             import xml.etree.ElementTree as ET
             for e in xml_data:
@@ -134,7 +144,7 @@ def accumulate_result(self, test, result):
                     print(xml_data, file=sys.__stderr__)
                     raise
 
-    def display_progress(self, test_index, test):
+    def display_progress(self, test_index, text):
         if self.ns.quiet:
             return
 
@@ -143,12 +153,12 @@ def display_progress(self, test_index, test):
         fails = len(self.bad) + len(self.environment_changed)
         if fails and not self.ns.pgo:
             line = f"{line}/{fails}"
-        line = f"[{line}] {test}"
+        line = f"[{line}] {text}"
 
         # add the system load prefix: "load avg: 1.80 "
-        if hasattr(os, 'getloadavg'):
-            load_avg_1min = os.getloadavg()[0]
-            line = f"load avg: {load_avg_1min:.2f} {line}"
+        load_avg = self.getloadavg()
+        if load_avg is not None:
+            line = f"load avg: {load_avg:.2f} {line}"
 
         # add the timestamp prefix:  "0:01:05 "
         test_time = time.monotonic() - self.start_time
@@ -164,22 +174,6 @@ def parse_args(self, kwargs):
                   "faulthandler.dump_traceback_later", file=sys.stderr)
             ns.timeout = None
 
-        if ns.threshold is not None and gc is None:
-            print('No GC available, ignore --threshold.', file=sys.stderr)
-            ns.threshold = None
-
-        if ns.findleaks:
-            if gc is not None:
-                # Uncomment the line below to report garbage that is not
-                # freeable by reference counting alone.  By default only
-                # garbage that is not collectable by the GC is reported.
-                pass
-                #gc.set_debug(gc.DEBUG_SAVEALL)
-            else:
-                print('No GC available, disabling --findleaks',
-                      file=sys.stderr)
-                ns.findleaks = False
-
         if ns.xmlpath:
             support.junit_xml_list = self.testsuite_xml = []
 
@@ -275,13 +269,13 @@ def list_cases(self):
         support.verbose = False
         support.set_match_tests(self.ns.match_tests)
 
-        for test in self.selected:
-            abstest = get_abs_module(self.ns, test)
+        for test_name in self.selected:
+            abstest = get_abs_module(self.ns, test_name)
             try:
                 suite = unittest.defaultTestLoader.loadTestsFromName(abstest)
                 self._list_cases(suite)
             except unittest.SkipTest:
-                self.skipped.append(test)
+                self.skipped.append(test_name)
 
         if self.skipped:
             print(file=sys.stderr)
@@ -298,23 +292,19 @@ def rerun_failed_tests(self):
         print()
         print("Re-running failed tests in verbose mode")
         self.rerun = self.bad[:]
-        for test in self.rerun:
-            print("Re-running test %r in verbose mode" % test, flush=True)
-            try:
-                self.ns.verbose = True
-                ok = runtest(self.ns, test)
-            except KeyboardInterrupt:
-                self.interrupted = True
-                # print a newline separate from the ^C
-                print()
+        for test_name in self.rerun:
+            print(f"Re-running {test_name} in verbose mode", flush=True)
+            self.ns.verbose = True
+            result = runtest(self.ns, test_name)
+
+            self.accumulate_result(result, rerun=True)
+
+            if result.result == INTERRUPTED:
                 break
-            else:
-                if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}:
-                    self.bad.remove(test)
-        else:
-            if self.bad:
-                print(count(len(self.bad), 'test'), "failed again:")
-                printlist(self.bad)
+
+        if self.bad:
+            print(count(len(self.bad), 'test'), "failed again:")
+            printlist(self.bad)
 
         self.display_result()
 
@@ -327,11 +317,11 @@ def display_result(self):
         print("== Tests result: %s ==" % self.get_tests_result())
 
         if self.interrupted:
-            print()
-            # print a newline after ^C
             print("Test suite interrupted by signal SIGINT.")
-            executed = set(self.good) | set(self.bad) | set(self.skipped)
-            omitted = set(self.selected) - executed
+
+        omitted = set(self.selected) - self.get_executed()
+        if omitted:
+            print()
             print(count(len(omitted), "test"), "omitted:")
             printlist(omitted)
 
@@ -348,8 +338,8 @@ def display_result(self):
             self.test_times.sort(reverse=True)
             print()
             print("10 slowest tests:")
-            for time, test in self.test_times[:10]:
-                print("- %s: %s" % (test, format_duration(time)))
+            for test_time, test in self.test_times[:10]:
+                print("- %s: %s" % (test, format_duration(test_time)))
 
         if self.bad:
             print()
@@ -387,50 +377,37 @@ def run_tests_sequential(self):
         print("Run tests sequentially")
 
         previous_test = None
-        for test_index, test in enumerate(self.tests, 1):
+        for test_index, test_name in enumerate(self.tests, 1):
             start_time = time.monotonic()
 
-            text = test
+            text = test_name
             if previous_test:
                 text = '%s -- %s' % (text, previous_test)
             self.display_progress(test_index, text)
 
             if self.tracer:
                 # If we're tracing code coverage, then we don't exit with status
                 # if on a false return value from main.
-                cmd = ('result = runtest(self.ns, test); '
-                       'self.accumulate_result(test, result)')
+                cmd = ('result = runtest(self.ns, test_name); '
+                       'self.accumulate_result(result)')
                 ns = dict(locals())
                 self.tracer.runctx(cmd, globals=globals(), locals=ns)
                 result = ns['result']
             else:
-                try:
-                    result = runtest(self.ns, test)
-                except KeyboardInterrupt:
-                    self.interrupted = True
-                    self.accumulate_result(test, (INTERRUPTED, None, None))
-                    break
-                else:
-                    self.accumulate_result(test, result)
-
-            previous_test = format_test_result(test, result[0])
+                result = runtest(self.ns, test_name)
+                self.accumulate_result(result)
+
+            if result.result == INTERRUPTED:
+                break
+
+            previous_test = format_test_result(result)
             test_time = time.monotonic() - start_time
             if test_time >= PROGRESS_MIN_TIME:
                 previous_test = "%s in %s" % (previous_test, format_duration(test_time))
             elif result[0] == PASSED:
                 # be quiet: say nothing if the test passed shortly
                 previous_test = None
 
-            if self.ns.findleaks:
-                gc.collect()
-                if gc.garbage:
-                    print("Warning: test created", len(gc.garbage), end=' ')
-                    print("uncollectable object(s).")
-                    # move the uncollectable objects somewhere so we don't see
-                    # them again
-                    self.found_garbage.extend(gc.garbage)
-                    del gc.garbage[:]
-
             # Unload the newly imported modules (best effort finalization)
             for module in sys.modules.keys():
                 if module not in save_modules and module.startswith("test."):
@@ -441,8 +418,8 @@ def run_tests_sequential(self):
 
     def _test_forever(self, tests):
         while True:
-            for test in tests:
-                yield test
+            for test_name in tests:
+                yield test_name
                 if self.bad:
                     return
                 if self.ns.fail_env_changed and self.environment_changed:
@@ -515,6 +492,10 @@ def run_tests(self):
             self.run_tests_sequential()
 
     def finalize(self):
+        if self.win_load_tracker is not None:
+            self.win_load_tracker.close()
+            self.win_load_tracker = None
+
         if self.next_single_filename:
             if self.next_single_test:
                 with open(self.next_single_filename, 'w') as fp:
@@ -585,6 +566,15 @@ def main(self, tests=None, **kwargs):
         with support.temp_cwd(test_cwd, quiet=True):
             self._main(tests, kwargs)
 
+    def getloadavg(self):
+        if self.win_load_tracker is not None:
+            return self.win_load_tracker.getloadavg()
+
+        if hasattr(os, 'getloadavg'):
+            return os.getloadavg()[0]
+
+        return None
+
     def _main(self, tests, kwargs):
         if self.ns.huntrleaks:
             warmup, repetitions, _ = self.ns.huntrleaks
@@ -616,6 +606,18 @@ def _main(self, tests, kwargs):
             self.list_cases()
             sys.exit(0)
 
+        # If we're on windows and this is the parent runner (not a worker),
+        # track the load average.
+        if sys.platform == 'win32' and (self.ns.worker_args is None):
+            from test.libregrtest.win_utils import WindowsLoadTracker
+
+            try:
+                self.win_load_tracker = WindowsLoadTracker()
+            except FileNotFoundError as error:
+                # Windows IoT Core and Windows Nano Server do not provide
+                # typeperf.exe for x64, x86 or ARM
+                print(f'Failed to create WindowsLoadTracker: {error}')
+
         self.run_tests()
         self.display_result()
 

Lib/test/libregrtest/refleak.py

@@ -1,4 +1,3 @@
-import errno
 import os
 import re
 import sys
@@ -18,7 +17,7 @@ def _get_dump(cls):
                 cls._abc_negative_cache, cls._abc_negative_cache_version)
 
 
-def dash_R(ns, the_module, test_name, test_func):
+def dash_R(ns, test_name, test_func):
     """Run a test multiple times, looking for reference leaks.
 
     Returns: