gh-111520: Integrate the Tier 2 interpreter in the Tier 1 interpreter

Python/ceval.c

@@ -201,15 +201,15 @@ maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, _PyInterpreterFrame *skip
     if (r < 0) {
         return -1;
     }
-    int lltrace = r;
+    int lltrace = r * 5;  // Levels 1-4 only trace uops
     if (!lltrace) {
-        // When tracing executed uops, also trace bytecode
-        char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
-        if (uop_debug != NULL && *uop_debug >= '0') {
-            lltrace = (*uop_debug - '0') >= 5;  // TODO: Parse an int and all that
+        // Can also be controlled by environment variable
+        char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
+        if (python_lltrace != NULL && *python_lltrace >= '0') {
+            lltrace = *python_lltrace - '0';  // TODO: Parse an int and all that
         }
     }
-    if (lltrace) {
+    if (lltrace >= 5) {
         lltrace_resume_frame(frame);
     }
     return lltrace;
@@ -679,9 +679,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 #ifdef Py_STATS
     int lastopcode = 0;
 #endif
-    // opcode is an 8-bit value to improve the code generated by MSVC
-    // for the big switch below (in combination with the EXTRA_CASES macro).
-    uint8_t opcode;        /* Current opcode */
+    int opcode;        /* Current opcode */
     int oparg;         /* Current opcode argument, if any */
 #ifdef LLTRACE
     int lltrace = 0;
@@ -729,6 +727,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
         goto resume_with_error;
     }
 
+    /* State shared between Tier 1 and Tier 2 interpreter */
+    _PyUOpExecutorObject *self = NULL;
+
     /* Local "register" variables.
      * These are cached values from the frame and code object.  */
 
@@ -765,7 +766,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     /* Start instructions */
 #if !USE_COMPUTED_GOTOS
     dispatch_opcode:
-        switch (opcode)
+        // Cast to an 8-bit value to improve the code generated by MSVC
+        // (in combination with the EXTRA_CASES macro).
+        switch ((uint8_t)opcode)
 #endif
         {
 
@@ -913,7 +916,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
             }
             /* Resume normal execution */
 #ifdef LLTRACE
-            if (lltrace) {
+            if (lltrace >= 5) {
                 lltrace_resume_frame(frame);
             }
 #endif
@@ -942,6 +945,123 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     stack_pointer = _PyFrame_GetStackPointer(frame);
     goto error;
 
+
+
+// The Tier 2 interpreter is also here!
+enter_tier_two:
+
+#undef LOAD_IP
+#define LOAD_IP(UNUSED) \
+do { ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; } while (0)
+
+#undef GOTO_ERROR
+#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two
+
+#undef DEOPT_IF
+#define DEOPT_IF(COND, INSTNAME) \
+    if ((COND)) {                \
+        goto deoptimize_tier_two;\
+    }
+
+#ifdef Py_STATS
+// Disable these macros that apply to Tier 1 stats when we are in Tier 2
+#undef STAT_INC
+#define STAT_INC(opname, name) ((void)0)
+#undef STAT_DEC
+#define STAT_DEC(opname, name) ((void)0)
+#undef CALL_STAT_INC
+#define CALL_STAT_INC(name) ((void)0)
+#endif
+
+#undef ENABLE_SPECIALIZATION
+#define ENABLE_SPECIALIZATION 0
+
+#ifdef Py_DEBUG
+    #define DPRINTF(level, ...) \
+        if (lltrace >= (level)) { printf(__VA_ARGS__); }
+#else
+    #define DPRINTF(level, ...)
+#endif
+
+    CHECK_EVAL_BREAKER();
+
+    OPT_STAT_INC(traces_executed);
+    _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
+    _PyUOpInstruction *next_uop = self->trace;
+    uint64_t operand;
+#ifdef Py_STATS
+    uint64_t trace_uop_execution_counter = 0;
+#endif
+
+    for (;;) {
+        opcode = next_uop->opcode;
+        oparg = next_uop->oparg;
+        operand = next_uop->operand;
+        DPRINTF(3,
+                "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n",
+                (int)(next_uop - self->trace),
+                opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode],
+                oparg,
+                operand,
+                (int)(stack_pointer - _PyFrame_Stackbase(frame)));
+        next_uop++;
+        OPT_STAT_INC(uops_executed);
+        UOP_EXE_INC(opcode);
+#ifdef Py_STATS
+        trace_uop_execution_counter++;
+#endif
+
+        switch (opcode) {
+
+#undef TIER_ONE
+#define TIER_TWO 2
+#include "executor_cases.c.h"
+
+            default:
+            {
+                fprintf(stderr, "Unknown uop %d, oparg %d, operand %" PRIu64 "\n",
+                        opcode, oparg, operand);
+                Py_FatalError("Unknown uop");
+            }
+
+        }
+    }
+
+unbound_local_error_tier_two:
+    _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
+        UNBOUNDLOCAL_ERROR_MSG,
+        PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
+    );
+    goto error_tier_two;
+
+pop_4_error_tier_two:
+    STACK_SHRINK(1);
+pop_3_error_tier_two:
+    STACK_SHRINK(1);
+pop_2_error_tier_two:
+    STACK_SHRINK(1);
+pop_1_error_tier_two:
+    STACK_SHRINK(1);
+error_tier_two:
+    DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
+    OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
+    frame->return_offset = 0;  // Don't leave this random
+    _PyFrame_SetStackPointer(frame, stack_pointer);
+    Py_DECREF(self);
+    goto resume_with_error;
+
+deoptimize_tier_two:
+    // On DEOPT_IF we just repeat the last instruction.
+    // This presumes nothing was popped from the stack (nor pushed).
+    DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
+    OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
+    frame->return_offset = 0;  // Dispatch to frame->instr_ptr
+    _PyFrame_SetStackPointer(frame, stack_pointer);
+    Py_DECREF(self);
+enter_tier_one:
+    next_instr = frame->instr_ptr;
+    goto resume_frame;
+
 }
 #if defined(__GNUC__)
 #  pragma GCC diagnostic pop

Python/ceval_macros.h

@@ -84,7 +84,7 @@
 
 /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */
 #ifdef LLTRACE
-#define PRE_DISPATCH_GOTO() if (lltrace) { \
+#define PRE_DISPATCH_GOTO() if (lltrace >= 5) { \
     lltrace_instruction(frame, stack_pointer, next_instr); }
 #else
 #define PRE_DISPATCH_GOTO() ((void)0)
@@ -116,11 +116,14 @@
         goto start_frame;                               \
     } while (0)
 
+// Use this instead of 'goto error' so Tier 2 can go to a different label
+#define GOTO_ERROR(LABEL) goto LABEL
+
 #define CHECK_EVAL_BREAKER() \
     _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \
     if (_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & _PY_EVAL_EVENTS_MASK) { \
         if (_Py_HandlePending(tstate) != 0) { \
-            goto error; \
+            GOTO_ERROR(error); \
         } \
     }
 
@@ -326,7 +329,7 @@ do { \
     }\
     else { \
         result = PyFloat_FromDouble(dval); \
-        if ((result) == NULL) goto error; \
+        if ((result) == NULL) GOTO_ERROR(error); \
         _Py_DECREF_NO_DEALLOC(left); \
         _Py_DECREF_NO_DEALLOC(right); \
     } \

Python/executor.c

@@ -21,11 +21,13 @@
 #define TIER_TWO 2
 #include "ceval_macros.h"
 
+#undef GOTO_ERROR
+#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two
 
 #undef DEOPT_IF
 #define DEOPT_IF(COND, INSTNAME) \
     if ((COND)) {                \
-        goto deoptimize;         \
+        goto deoptimize_tier_two;\
     }
 
 #ifdef Py_STATS
@@ -45,11 +47,12 @@
 _PyInterpreterFrame *
 _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer)
 {
+    Py_FatalError("Tier 2 is now inlined into Tier 1");
 #ifdef Py_DEBUG
-    char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
+    char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
     int lltrace = 0;
-    if (uop_debug != NULL && *uop_debug >= '0') {
-        lltrace = *uop_debug - '0';  // TODO: Parse an int and all that
+    if (python_lltrace != NULL && *python_lltrace >= '0') {
+        lltrace = *python_lltrace - '0';  // TODO: Parse an int and all that
     }
     #define DPRINTF(level, ...) \
         if (lltrace >= (level)) { printf(__VA_ARGS__); }
@@ -72,7 +75,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
 
     OPT_STAT_INC(traces_executed);
     _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
-    int pc = 0;
+    _PyUOpInstruction *next_uop = self->trace;
     int opcode;
     int oparg;
     uint64_t operand;
@@ -81,17 +84,17 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
 #endif
 
     for (;;) {
-        opcode = self->trace[pc].opcode;
-        oparg = self->trace[pc].oparg;
-        operand = self->trace[pc].operand;
+        opcode = next_uop->opcode;
+        oparg = next_uop->oparg;
+        operand = next_uop->operand;
         DPRINTF(3,
                 "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n",
-                pc,
+                (int)(next_uop - self->trace),
                 opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode],
                 oparg,
                 operand,
                 (int)(stack_pointer - _PyFrame_Stackbase(frame)));
-        pc++;
+        next_uop++;
         OPT_STAT_INC(uops_executed);
         UOP_EXE_INC(opcode);
 #ifdef Py_STATS
@@ -111,22 +114,22 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
         }
     }
 
-unbound_local_error:
+unbound_local_error_tier_two:
     _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
         UNBOUNDLOCAL_ERROR_MSG,
         PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
     );
-    goto error;
+    goto error_tier_two;
 
-pop_4_error:
+pop_4_error_tier_two:
     STACK_SHRINK(1);
-pop_3_error:
+pop_3_error_tier_two:
     STACK_SHRINK(1);
-pop_2_error:
+pop_2_error_tier_two:
     STACK_SHRINK(1);
-pop_1_error:
+pop_1_error_tier_two:
     STACK_SHRINK(1);
-error:
+error_tier_two:
     // On ERROR_IF we return NULL as the frame.
     // The caller recovers the frame from tstate->current_frame.
     DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
@@ -136,13 +139,14 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject
     Py_DECREF(self);
     return NULL;
 
-deoptimize:
+deoptimize_tier_two:
     // On DEOPT_IF we just repeat the last instruction.
     // This presumes nothing was popped from the stack (nor pushed).
     DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
     OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
     frame->return_offset = 0;  // Dispatch to frame->instr_ptr
     _PyFrame_SetStackPointer(frame, stack_pointer);
     Py_DECREF(self);
+enter_tier_one:
     return frame;
 }