pythongh-110152: regrtest handles cross compilation and HOSTRUNNER

* _add_python_opts() now handles cross compilation and HOSTRUNNER.
* display_header() now tells if Python is cross-compiled, display
  HOSTRUNNER, and get the host platform.
* Remove Tools/scripts/run_tests.py script.
* Remove "make hostrunnertest": use "make buildbottest"
  or "make test" instead.

Lib/test/libregrtest/main.py

@@ -3,6 +3,7 @@
 import re
 import shlex
 import sys
+import sysconfig
 import time
 
 from test import support
@@ -22,6 +23,7 @@
     strip_py_suffix, count, format_duration,
     printlist, get_temp_dir, get_work_dir, exit_timeout,
     display_header, cleanup_temp_dir, print_warning,
+    is_cross_compiled, get_host_runner,
     MS_WINDOWS, EXIT_TIMEOUT)
 
 
@@ -71,10 +73,9 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False):
         self.want_rerun: bool = ns.rerun
         self.want_run_leaks: bool = ns.runleaks
 
-        ci_mode = (ns.fast_ci or ns.slow_ci)
+        self.ci_mode: bool = (ns.fast_ci or ns.slow_ci)
         self.want_add_python_opts: bool = (_add_python_opts
-                                           and ns._add_python_opts
-                                           and ci_mode)
+                                           and ns._add_python_opts)
 
         # Select tests
         if ns.match_tests:
@@ -431,7 +432,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
         if (self.want_header
             or not(self.pgo or self.quiet or self.single_test_run
                    or tests or self.cmdline_args)):
-            display_header(self.use_resources)
+            display_header(self.use_resources, self.python_cmd)
 
         if self.randomize:
             print("Using random seed", self.random_seed)
@@ -489,8 +490,56 @@ def run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
                 # processes.
                 return self._run_tests(selected, tests)
 
-    def _add_python_opts(self):
-        python_opts = []
+    def _add_cross_compile_opts(self, regrtest_opts):
+        # WASM/WASI buildbot builders pass multiple PYTHON environment
+        # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER.
+        keep_environ = bool(self.python_cmd)
+        environ = None
+
+        # Are we using cross-compilation?
+        cross_compile = is_cross_compiled()
+
+        # Get HOSTRUNNER
+        hostrunner = get_host_runner()
+
+        if cross_compile:
+            # emulate -E, but keep PYTHONPATH + cross compile env vars,
+            # so test executable can load correct sysconfigdata file.
+            keep = {
+                '_PYTHON_PROJECT_BASE',
+                '_PYTHON_HOST_PLATFORM',
+                '_PYTHON_SYSCONFIGDATA_NAME',
+                'PYTHONPATH'
+            }
+            old_environ = os.environ
+            new_environ = {
+                name: value for name, value in os.environ.items()
+                if not name.startswith(('PYTHON', '_PYTHON')) or name in keep
+            }
+            # Only set environ if at least one variable was removed
+            if new_environ != old_environ:
+                environ = new_environ
+            keep_environ = True
+
+        if cross_compile and hostrunner:
+            if self.num_workers == 0:
+                # For now use only two cores for cross-compiled builds;
+                # hostrunner can be expensive.
+                regrtest_opts.extend(['-j', '2'])
+
+            # If HOSTRUNNER is set and -p/--python option is not given, then
+            # use hostrunner to execute python binary for tests.
+            if not self.python_cmd:
+                buildpython = sysconfig.get_config_var("BUILDPYTHON")
+                python_cmd = f"{hostrunner} {buildpython}"
+                regrtest_opts.extend(["--python", python_cmd])
+                keep_environ = True
+
+        return (environ, keep_environ)
+
+    def _add_ci_python_opts(self, python_opts, keep_environ):
+        # --fast-ci and --slow-ci add options to Python:
+        # "-u -W default -bb -E"
 
         # Unbuffered stdout and stderr
         if not sys.stdout.write_through:
@@ -504,32 +553,27 @@ def _add_python_opts(self):
         if sys.flags.bytes_warning < 2:
             python_opts.append('-bb')
 
-        # WASM/WASI buildbot builders pass multiple PYTHON environment
-        # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER.
-        if not self.python_cmd:
+        if not keep_environ:
             # Ignore PYTHON* environment variables
             if not sys.flags.ignore_environment:
                 python_opts.append('-E')
 
-        if not python_opts:
-            return
-
-        cmd = [*sys.orig_argv, "--dont-add-python-opts"]
-        cmd[1:1] = python_opts
-
+    def _execute_python(self, cmd, environ):
         # Make sure that messages before execv() are logged
         sys.stdout.flush()
         sys.stderr.flush()
 
         cmd_text = shlex.join(cmd)
         try:
+            print(f"+ {cmd_text}", flush=True)
+
             if hasattr(os, 'execv') and not MS_WINDOWS:
                 os.execv(cmd[0], cmd)
                 # On success, execv() do no return.
                 # On error, it raises an OSError.
             else:
                 import subprocess
-                with subprocess.Popen(cmd) as proc:
+                with subprocess.Popen(cmd, env=environ) as proc:
                     try:
                         proc.wait()
                     except KeyboardInterrupt:
@@ -548,6 +592,28 @@ def _add_python_opts(self):
                           f"Command: {cmd_text}")
             # continue executing main()
 
+    def _add_python_opts(self):
+        python_opts = []
+        regrtest_opts = []
+
+        environ, keep_environ = self._add_cross_compile_opts(regrtest_opts)
+        if self.ci_mode:
+            self._add_ci_python_opts(python_opts, keep_environ)
+
+        if (not python_opts) and (not regrtest_opts) and (environ is None):
+            # Nothing changed: nothing to do
+            return
+
+        # Create new command line
+        cmd = list(sys.orig_argv)
+        if python_opts:
+            cmd[1:1] = python_opts
+        if regrtest_opts:
+            cmd.extend(regrtest_opts)
+        cmd.append("--dont-add-python-opts")
+
+        self._execute_python(cmd, environ)
+
     def _init(self):
         # Set sys.stdout encoder error handler to backslashreplace,
         # similar to sys.stderr error handler, to avoid UnicodeEncodeError

Lib/test/libregrtest/utils.py

@@ -5,7 +5,9 @@
 import os.path
 import platform
 import random
+import shlex
 import signal
+import subprocess
 import sys
 import sysconfig
 import tempfile
@@ -523,7 +525,18 @@ def adjust_rlimit_nofile():
                           f"{new_fd_limit}: {err}.")
 
 
-def display_header(use_resources: tuple[str, ...]):
+def get_host_runner():
+    if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None:
+        hostrunner = sysconfig.get_config_var("HOSTRUNNER")
+    return hostrunner
+
+
+def is_cross_compiled():
+    return ('_PYTHON_HOST_PLATFORM' in os.environ)
+
+
+def display_header(use_resources: tuple[str, ...],
+                   python_cmd: tuple[str, ...] | None):
     # Print basic platform information
     print("==", platform.python_implementation(), *sys.version.split())
     print("==", platform.platform(aliased=True),
@@ -537,13 +550,35 @@ def display_header(use_resources: tuple[str, ...]):
     print("== encodings: locale=%s, FS=%s"
           % (locale.getencoding(), sys.getfilesystemencoding()))
 
-
     if use_resources:
         print(f"== resources ({len(use_resources)}): "
               f"{', '.join(sorted(use_resources))}")
     else:
         print("== resources: (all disabled, use -u option)")
 
+    cross_compile = is_cross_compiled()
+    if cross_compile:
+        print("== cross compiled: Yes")
+    if python_cmd:
+        cmd = shlex.join(python_cmd)
+        print(f"== host python: {cmd}")
+
+        get_cmd = [*python_cmd, '-m', 'platform']
+        proc = subprocess.run(
+            get_cmd,
+            stdout=subprocess.PIPE,
+            text=True,
+            cwd=os_helper.SAVEDCWD)
+        stdout = proc.stdout.replace('\n', ' ').strip()
+        if stdout:
+            print(f"== host platform: {stdout}")
+        elif proc.returncode:
+            print(f"== host platform: <command failed with exit code {proc.returncode}>")
+    else:
+        hostrunner = get_host_runner()
+        if hostrunner:
+            print(f"== host runner: {hostrunner}")
+
     # This makes it easier to remember what to set in your local
     # environment when trying to reproduce a sanitizer failure.
     asan = support.check_sanitizer(address=True)

Lib/test/test_regrtest.py

@@ -788,14 +788,6 @@ def test_script_autotest(self):
         args = [*self.python_args, script, *self.regrtest_args, *self.tests]
         self.run_tests(args)
 
-    @unittest.skipUnless(sysconfig.is_python_build(),
-                         'run_tests.py script is not installed')
-    def test_tools_script_run_tests(self):
-        # Tools/scripts/run_tests.py
-        script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py')
-        args = [script, *self.regrtest_args, *self.tests]
-        self.run_tests(args)
-
     def run_batch(self, *args):
         proc = self.run_command(args)
         self.check_output(proc.stdout)

Makefile.pre.in

@@ -1837,7 +1837,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS)
 
 TESTOPTS=	$(EXTRATESTOPTS)
 TESTPYTHON=	$(RUNSHARED) $(PYTHON_FOR_BUILD) $(TESTPYTHONOPTS)
-TESTRUNNER=	$(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py
+TESTRUNNER=	$(TESTPYTHON) -m test
 TESTTIMEOUT=
 
 # Remove "test_python_*" directories of previous failed test jobs.
@@ -1875,11 +1875,6 @@ buildbottest: all
 	fi
 	$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
 
-# Like buildbottest, but run Python tests with HOSTRUNNER directly.
-.PHONY: hostrunnertest
-hostrunnertest: all
-	$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
-
 .PHONY: pythoninfo
 pythoninfo: all
 		$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo

Misc/NEWS.d/next/Tests/2023-09-30-20-18-38.gh-issue-110152.4Kxve1.rst

@@ -0,0 +1,5 @@
+Remove ``Tools/scripts/run_tests.py`` and ``make hostrunnertest``. Just run
+``./python -m test --slow-ci``, ``make buildbottest`` or ``make test`` instead.
+Python test runner (regrtest) now handles cross-compilation and HOSTRUNNER. It
+also adds options to Python such fast ``-u -E -W default -bb`` when
+``--fast-ci`` or ``--slow-ci`` option is used. Patch by Victor Stinner.