gh-111520: Integrate the Tier 2 interpreter in the Tier 1 interpreter (…

…#111428)

- There is no longer a separate Python/executor.c file.
- Conventions in Python/bytecodes.c are slightly different -- don't use `goto error`,
  you must use `GOTO_ERROR(error)` (same for others like `unused_local_error`).
- The `TIER_ONE` and `TIER_TWO` symbols are only valid in the generated (.c.h) files.
- In Lib/test/support/__init__.py, `Py_C_RECURSION_LIMIT` is imported from `_testcapi`.
- On Windows, in debug mode, stack allocation grows from 8MiB to 12MiB.
- **Beware!** This changes the env vars to enable uops and their debugging
  to `PYTHON_UOPS` and `PYTHON_LLTRACE`.

Lib/test/support/__init__.py

@@ -2355,8 +2355,15 @@ def adjust_int_max_str_digits(max_digits):
 #For recursion tests, easily exceeds default recursion limit
 EXCEEDS_RECURSION_LIMIT = 5000
 
-# The default C recursion limit (from Include/cpython/pystate.h).
-Py_C_RECURSION_LIMIT = 1500
+def _get_c_recursion_limit():
+    try:
+        import _testcapi
+        return _testcapi.Py_C_RECURSION_LIMIT
+    except (ImportError, AttributeError):
+        return 1500  #  (from Include/cpython/pystate.h)
+
+# The default C recursion limit.
+Py_C_RECURSION_LIMIT = _get_c_recursion_limit()
 
 #Windows doesn't have os.uname() but it doesn't support s390x.
 skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',

Lib/test/test_generated_cases.py

@@ -118,17 +118,19 @@ def run_cases_test(self, input: str, expected: str):
 
         with open(self.temp_output_filename) as temp_output:
             lines = temp_output.readlines()
-            while lines and lines[0].startswith("// "):
+            while lines and lines[0].startswith(("// ", "#", "    #", "\n")):
                 lines.pop(0)
+            while lines and lines[-1].startswith(("#", "\n")):
+                lines.pop(-1)
         actual = "".join(lines)
-        # if actual.rstrip() != expected.rstrip():
+        # if actual.strip() != expected.strip():
         #     print("Actual:")
         #     print(actual)
         #     print("Expected:")
         #     print(expected)
         #     print("End")
 
-        self.assertEqual(actual.rstrip(), expected.rstrip())
+        self.assertEqual(actual.strip(), expected.strip())
 
     def test_inst_no_args(self):
         input = """

Makefile.pre.in

@@ -412,7 +412,6 @@ PYTHON_OBJS=	\
 		Python/crossinterp.o \
 		Python/dynamic_annotations.o \
 		Python/errors.o \
-		Python/executor.o \
 		Python/flowgraph.o \
 		Python/frame.o \
 		Python/frozenmain.o \
@@ -1603,12 +1602,6 @@ Python/ceval.o: \
 		$(srcdir)/Python/generated_cases.c.h \
 		$(srcdir)/Python/opcode_targets.h
 
-Python/executor.o: \
-		$(srcdir)/Include/internal/pycore_opcode_metadata.h \
-		$(srcdir)/Include/internal/pycore_optimizer.h \
-		$(srcdir)/Python/ceval_macros.h \
-		$(srcdir)/Python/executor_cases.c.h
-
 Python/flowgraph.o: \
 		$(srcdir)/Include/internal/pycore_opcode_metadata.h
 

Misc/NEWS.d/next/Core and Builtins/2023-10-31-21-33-35.gh-issue-111520.vw-rxJ.rst

@@ -0,0 +1,9 @@
+Merge the Tier 1 (bytecode) and Tier 2 (micro-ops) interpreters together,
+moving the Tier 2 interpreter loop and switch into
+``_PyEval_EvalFrameDefault()`` in ``Python/ceval.c``.
+The ``Python/executor.c`` file is gone.
+Also the ``TIER_ONE`` and ``TIER_TWO`` macros are now handled
+by the code generator.
+
+**Beware!** This changes the environment variables to enable micro-ops and
+their debugging to ``PYTHON_UOPS`` and ``PYTHON_LLTRACE``.

PCbuild/_freeze_module.vcxproj

@@ -200,7 +200,6 @@
     <ClCompile Include="..\Python\dynamic_annotations.c" />
     <ClCompile Include="..\Python\dynload_win.c" />
     <ClCompile Include="..\Python\errors.c" />
-    <ClCompile Include="..\Python\executor.c" />
     <ClCompile Include="..\Python\fileutils.c" />
     <ClCompile Include="..\Python\flowgraph.c" />
     <ClCompile Include="..\Python\formatter_unicode.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -127,9 +127,6 @@
     <ClCompile Include="..\Python\errors.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Python\executor.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Objects\exceptions.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/python.vcxproj

@@ -95,7 +95,7 @@
     <Link>
       <SubSystem>Console</SubSystem>
       <StackReserveSize Condition="$(Configuration) != 'Debug'">2000000</StackReserveSize>
-      <StackReserveSize Condition="$(Configuration) == 'Debug'">8000000</StackReserveSize>
+      <StackReserveSize Condition="$(Configuration) == 'Debug'">12000000</StackReserveSize>
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>

PCbuild/pythoncore.vcxproj

@@ -552,7 +552,6 @@
     <ClCompile Include="..\Python\dynamic_annotations.c" />
     <ClCompile Include="..\Python\dynload_win.c" />
     <ClCompile Include="..\Python\errors.c" />
-    <ClCompile Include="..\Python\executor.c" />
     <ClCompile Include="..\Python\fileutils.c" />
     <ClCompile Include="..\Python\flowgraph.c" />
     <ClCompile Include="..\Python\formatter_unicode.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -1253,9 +1253,6 @@
     <ClCompile Include="..\Python\errors.c">
       <Filter>Python</Filter>
     </ClCompile>
-    <ClCompile Include="..\Python\executor.c">
-      <Filter>Python</Filter>
-    </ClCompile>
     <ClCompile Include="..\Python\fileutils.c">
       <Filter>Python</Filter>
     </ClCompile>