Jump directly to side-exit executors

Include/cpython/optimizer.h

@@ -61,7 +61,7 @@ PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void);
 PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset);
 
 int _PyOptimizer_BackEdge(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer);
-int _PyOptimizer_Anywhere(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer);
+int _PyOptimizer_Unanchored(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **pexecutor, PyObject **stack_pointer);
 
 extern _PyOptimizerObject _PyOptimizer_Default;
 

Python/ceval.c

@@ -1108,9 +1108,22 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     _PyFrame_SetStackPointer(frame, stack_pointer);
     // Increment side exit counter for this uop
     int pc = next_uop - 1 - current_executor->trace;
+    _PyExecutorObject **pexecutor = current_executor->executors + pc;
+    if (*pexecutor != NULL) {
+        PyCodeObject *code = _PyFrame_GetCode(frame);
+        DPRINTF(2, "Jumping to new executor for %s (%s:%d) at byte offset %d\n",
+                PyUnicode_AsUTF8(code->co_qualname),
+                PyUnicode_AsUTF8(code->co_filename),
+                code->co_firstlineno,
+                2 * (int)(frame->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(frame))));
+        Py_DECREF(current_executor);
+        current_executor = (_PyUOpExecutorObject *)*pexecutor;
+        Py_INCREF(current_executor);
+        goto enter_tier_two;
+    }
     uint16_t *pcounter = current_executor->counters + pc;
     *pcounter += 1;
-    if (*pcounter == 16 &&  // TODO: use resume_threshold
+    if (*pcounter == 32 &&  // TODO: use resume_threshold
         tstate->interp->optimizer != &_PyOptimizer_Default &&
         (opcode == POP_JUMP_IF_FALSE ||
          opcode == POP_JUMP_IF_TRUE ||
@@ -1121,14 +1134,32 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
                 _PyUOpName(uopcode), pc, current_executor, (int)(*pcounter));
         DPRINTF(2, "    T1: %s\n", _PyOpcode_OpName[opcode]);
         // The counter will cycle around once the 16 bits overflow
-        int optimized = _PyOptimizer_Anywhere(frame, src, dest, stack_pointer);
+        int optimized = _PyOptimizer_Unanchored(frame, dest, pexecutor, stack_pointer);
         if (optimized < 0) {
             goto error_tier_two;
         }
         if (optimized) {
             DPRINTF(1, "--> Optimized %s @ %d in %p\n",
                     _PyUOpName(uopcode), pc, current_executor);
             DPRINTF(1, "    T1: %s\n", _PyOpcode_OpName[src->op.code]);
+            PyCodeObject *code = _PyFrame_GetCode(frame);
+            DPRINTF(2, "Jumping to fresh executor for %s (%s:%d) at byte offset %d\n",
+                    PyUnicode_AsUTF8(code->co_qualname),
+                    PyUnicode_AsUTF8(code->co_filename),
+                    code->co_firstlineno,
+                    2 * (int)(frame->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(frame))));
+            Py_DECREF(current_executor);
+            current_executor = (_PyUOpExecutorObject *)*pexecutor;
+            if (current_executor->trace[0].opcode != uopcode) {
+                Py_INCREF(current_executor);
+                goto enter_tier_two;
+            }
+            // This is guaranteed to deopt again; forget about it
+            DPRINTF(2, "It's not an improvement -- discarding trace\n");
+            *pexecutor = NULL;
+            Py_DECREF(current_executor);
+            next_instr = frame->instr_ptr;
+            goto resume_frame;
         }
         else {
             DPRINTF(2, "--> Failed to optimize %s @ %d in %p\n",

Python/optimizer.c

@@ -155,11 +155,10 @@ PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer)
     Py_DECREF(old);
 }
 
-// src is where to insert ENTER_EXECUTOR
-// dest is where to start tracing
-static int
-optimizer_wherever(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer)
+int
+_PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer)
 {
+    assert(src->op.code == JUMP_BACKWARD);
     PyCodeObject *code = _PyFrame_GetCode(frame);
     assert(PyCode_Check(code));
     PyInterpreterState *interp = _PyInterpreterState_GET();
@@ -189,27 +188,27 @@ optimizer_wherever(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *
     return 1;
 }
 
+// Return an unanchored executor. The caller owns the executor when returning 1.
+// No ENTER_EXECUTOR is inserted, nor is the executor added to the code object.
 int
-_PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer)
-{
-    assert(src->op.code == JUMP_BACKWARD);
-    return optimizer_wherever(frame, src, dest, stack_pointer);
-}
-
-// Start tracing and insert ENTER_EXECUTOR at the same place.
-// Normally src == dest, but when there's an EXTENDED_ARG involved,
-// dest points at the preceding EXTENDED_ARG.
-// Do not use at JUMP_BACKWARD.  Won't replace ENTER_EXECUTOR.
-int
-_PyOptimizer_Anywhere(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer)
+_PyOptimizer_Unanchored(
+    _PyInterpreterFrame *frame,
+    _Py_CODEUNIT *instr,
+    _PyExecutorObject **pexecutor,
+    PyObject **stack_pointer)
 {
-    if (src->op.code == JUMP_BACKWARD) {
+    if (instr->op.code == JUMP_BACKWARD || instr->op.code == ENTER_EXECUTOR) {
         return 0;
     }
-    if (src->op.code == ENTER_EXECUTOR) {
+    PyCodeObject *code = _PyFrame_GetCode(frame);
+    assert(PyCode_Check(code));
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    _PyOptimizerObject *opt = interp->optimizer;
+    if (strcmp(opt->ob_base.ob_type->tp_name, "uop_optimizer") != 0) {
         return 0;
     }
-    return optimizer_wherever(frame, src, dest, stack_pointer);
+    *pexecutor = NULL;
+    return opt->optimize(opt, code, instr, pexecutor, (int)(stack_pointer - _PyFrame_Stackbase(frame)));
 }
 
 _PyExecutorObject *