pythongh-110152: regrtest handles cross compilation and HOSTRUNNER

Lib/test/libregrtest/main.py

@@ -3,6 +3,7 @@
 import re
 import shlex
 import sys
+import sysconfig
 import time
 
 from test import support
@@ -71,10 +72,9 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False):
         self.want_rerun: bool = ns.rerun
         self.want_run_leaks: bool = ns.runleaks
 
-        ci_mode = (ns.fast_ci or ns.slow_ci)
+        self.ci_mode: bool = (ns.fast_ci or ns.slow_ci)
         self.want_add_python_opts: bool = (_add_python_opts
-                                           and ns._add_python_opts
-                                           and ci_mode)
+                                           and ns._add_python_opts)
 
         # Select tests
         if ns.match_tests:
@@ -489,8 +489,57 @@ def run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
                 # processes.
                 return self._run_tests(selected, tests)
 
-    def _add_python_opts(self):
-        python_opts = []
+    def _add_cross_compile_opts(self, regrtest_opts):
+        # WASM/WASI buildbot builders pass multiple PYTHON environment
+        # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER.
+        keep_environ = bool(self.python_cmd)
+        environ = None
+
+        # Are we using cross-compilation?
+        cross_compile = ('_PYTHON_HOST_PLATFORM' in os.environ)
+
+        # Get HOSTRUNNER
+        if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None:
+            hostrunner = sysconfig.get_config_var("HOSTRUNNER")
+
+        if cross_compile:
+            # emulate -E, but keep PYTHONPATH + cross compile env vars,
+            # so test executable can load correct sysconfigdata file.
+            keep = {
+                '_PYTHON_PROJECT_BASE',
+                '_PYTHON_HOST_PLATFORM',
+                '_PYTHON_SYSCONFIGDATA_NAME',
+                'PYTHONPATH'
+            }
+            old_environ = os.environ
+            new_environ = {
+                name: value for name, value in os.environ.items()
+                if not name.startswith(('PYTHON', '_PYTHON')) or name in keep
+            }
+            # Only set environ if at least one variable was removed
+            if new_environ != old_environ:
+                environ = new_environ
+            keep_environ = True
+
+        if cross_compile and hostrunner:
+            if self.num_workers == 0:
+                # For now use only two cores for cross-compiled builds;
+                # hostrunner can be expensive.
+                regrtest_opts.extend(['-j', '2'])
+
+            # If HOSTRUNNER is set and -p/--python option is not given, then
+            # use hostrunner to execute python binary for tests.
+            if not self.python_cmd:
+                buildpython = sysconfig.get_config_var("BUILDPYTHON")
+                python_cmd = f"{hostrunner} {buildpython}"
+                regrtest_opts.extend(["--python", python_cmd])
+                keep_environ = True
+
+        return (environ, keep_environ)
+
+    def _add_ci_python_opts(self, python_opts, keep_environ):
+        # --fast-ci and --slow-ci add options to Python:
+        # "-u -W default -bb -E"
 
         # Unbuffered stdout and stderr
         if not sys.stdout.write_through:
@@ -504,32 +553,27 @@ def _add_python_opts(self):
         if sys.flags.bytes_warning < 2:
             python_opts.append('-bb')
 
-        # WASM/WASI buildbot builders pass multiple PYTHON environment
-        # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER.
-        if not self.python_cmd:
+        if not keep_environ:
             # Ignore PYTHON* environment variables
             if not sys.flags.ignore_environment:
                 python_opts.append('-E')
 
-        if not python_opts:
-            return
-
-        cmd = [*sys.orig_argv, "--dont-add-python-opts"]
-        cmd[1:1] = python_opts
-
+    def _execute_python(self, cmd, environ):
         # Make sure that messages before execv() are logged
         sys.stdout.flush()
         sys.stderr.flush()
 
         cmd_text = shlex.join(cmd)
         try:
+            print(f"+ {cmd_text}", flush=True)
+
             if hasattr(os, 'execv') and not MS_WINDOWS:
                 os.execv(cmd[0], cmd)
                 # On success, execv() do no return.
                 # On error, it raises an OSError.
             else:
                 import subprocess
-                with subprocess.Popen(cmd) as proc:
+                with subprocess.Popen(cmd, env=environ) as proc:
                     try:
                         proc.wait()
                     except KeyboardInterrupt:
@@ -548,6 +592,29 @@ def _add_python_opts(self):
                           f"Command: {cmd_text}")
             # continue executing main()
 
+    def _add_python_opts(self):
+        python_opts = []
+        regrtest_opts = []
+
+        environ, keep_environ = self._add_cross_compile_opts(regrtest_opts)
+        if self.ci_mode:
+            self._add_ci_python_opts(python_opts, keep_environ)
+
+        if (not python_opts) and (not regrtest_opts) and (environ is None):
+            # Nothing changed: nothing to do
+            print("NOTHING TO DO")
+            return
+
+        # Create new command line
+        cmd = list(sys.orig_argv)
+        if python_opts:
+            cmd[1:1] = python_opts
+        if regrtest_opts:
+            cmd.extend(regrtest_opts)
+        cmd.append("--dont-add-python-opts")
+
+        self._execute_python(cmd, environ)
+
     def _init(self):
         # Set sys.stdout encoder error handler to backslashreplace,
         # similar to sys.stderr error handler, to avoid UnicodeEncodeError

Lib/test/test_regrtest.py

@@ -787,14 +787,6 @@ def test_script_autotest(self):
         args = [*self.python_args, script, *self.regrtest_args, *self.tests]
         self.run_tests(args)
 
-    @unittest.skipUnless(sysconfig.is_python_build(),
-                         'run_tests.py script is not installed')
-    def test_tools_script_run_tests(self):
-        # Tools/scripts/run_tests.py
-        script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py')
-        args = [script, *self.regrtest_args, *self.tests]
-        self.run_tests(args)
-
     def run_batch(self, *args):
         proc = self.run_command(args)
         self.check_output(proc.stdout)

Makefile.pre.in

@@ -1837,7 +1837,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS)
 
 TESTOPTS=	$(EXTRATESTOPTS)
 TESTPYTHON=	$(RUNSHARED) $(PYTHON_FOR_BUILD) $(TESTPYTHONOPTS)
-TESTRUNNER=	$(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py
+TESTRUNNER=	$(TESTPYTHON) -m test
 TESTTIMEOUT=
 
 # Remove "test_python_*" directories of previous failed test jobs.
@@ -1875,11 +1875,6 @@ buildbottest: all
 	fi
 	$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
 
-# Like buildbottest, but run Python tests with HOSTRUNNER directly.
-.PHONY: hostrunnertest
-hostrunnertest: all
-	$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
-
 .PHONY: pythoninfo
 pythoninfo: all
 		$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo

Misc/NEWS.d/next/Tests/2023-09-30-20-18-38.gh-issue-110152.4Kxve1.rst

@@ -0,0 +1,5 @@
+Remove ``Tools/scripts/run_tests.py`` and ``make hostrunnertest``. Just run
+``./python -m test --slow-ci`` or ``make buildbottest`` instead. Python test
+runner (regrtest) now handles cross-compilation and HOSTRUNNER. It also adds
+options to Python such fast ``-u -E -W default -bb`` when ``--fast-ci`` or
+``--slow-ci`` option is used. Patch by Victor Stinner.