fix: proper tracing of call/return for Python 3.11.0a4

Version 3.11.0a4 introduced RESUME, so returns and calls are different now.
This change also fixes some mishandling of yield-from in previous releases.

CHANGES.rst

@@ -22,7 +22,9 @@ This list is detailed and covers changes in each pre-release version.
 Unreleased
 ----------
 
-- Dropped support for Python 3.6, which ended support on 2021-12-23.
+- Dropped support for Python 3.6, which reached end-of-life on 2021-12-23.
+
+- Updated Python 3.11 support to 3.11.0a4.
 
 - Fix: a .gitignore file will only be written into the HTML report output
   directory if the directory is empty.  This should prevent certain unfortunate

coverage/ctracer/tracer.c

@@ -520,10 +520,24 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame)
     Py_XSETREF(frame->f_trace, (PyObject*)self);
 
     /* A call event is really a "start frame" event, and can happen for
-     * re-entering a generator also.  f_lasti is -1 for a true call, and a
-     * real byte offset for a generator re-entry.
+     * re-entering a generator also.  How we tell the difference depends on
+     * the version of Python.
      */
-    if (MyFrame_lasti(frame) < 0) {
+    BOOL real_call = FALSE;
+
+#ifdef RESUME   // 3.11.0a4
+    /*
+     * The current opcode is guaranteed to be RESUME. The argument
+     * determines what kind of resume it is.
+     */
+    PyObject * pCode = MyFrame_GetCode(frame)->co_code;
+    real_call = (PyBytes_AS_STRING(pCode)[MyFrame_lasti(frame) + 1] == 0);
+#else
+    // f_lasti is -1 for a true call, and a real byte offset for a generator re-entry.
+    real_call = (MyFrame_lasti(frame) < 0);
+#endif
+
+    if (real_call) {
         self->pcur_entry->last_line = -MyFrame_GetCode(frame)->co_firstlineno;
     }
     else {
@@ -683,19 +697,36 @@ CTracer_handle_return(CTracer *self, PyFrameObject *frame)
 
     if (self->pdata_stack->depth >= 0) {
         if (self->tracing_arcs && self->pcur_entry->file_data) {
+            BOOL real_return = FALSE;
+            PyObject * pCode = MyFrame_GetCode(frame)->co_code;
+            int lasti = MyFrame_lasti(frame);
+            Py_ssize_t code_size = PyBytes_GET_SIZE(pCode);
+            unsigned char * code_bytes = (unsigned char *)PyBytes_AS_STRING(pCode);
+#ifdef RESUME
+            if (lasti == code_size - 2) {
+                real_return = TRUE;
+            }
+            else {
+                real_return = (code_bytes[lasti + 2] != RESUME);
+            }
+#else
             /* Need to distinguish between RETURN_VALUE and YIELD_VALUE. Read
              * the current bytecode to see what it is.  In unusual circumstances
              * (Cython code), co_code can be the empty string, so range-check
              * f_lasti before reading the byte.
              */
-            int bytecode = RETURN_VALUE;
-            PyObject * pCode = MyFrame_GetCode(frame)->co_code;
-            int lasti = MyFrame_lasti(frame);
+            BOOL is_yield = FALSE;
+            BOOL is_yield_from = FALSE;
 
-            if (lasti < PyBytes_GET_SIZE(pCode)) {
-                bytecode = PyBytes_AS_STRING(pCode)[lasti];
+            if (lasti < code_size) {
+                is_yield = (code_bytes[lasti] == YIELD_VALUE);
+                if (lasti + 2 < code_size) {
+                    is_yield_from = (code_bytes[lasti + 2] == YIELD_FROM);
+                }
             }
-            if (bytecode != YIELD_VALUE) {
+            real_return = !(is_yield || is_yield_from);
+#endif
+            if (real_return) {
                 int first = MyFrame_GetCode(frame)->co_firstlineno;
                 if (CTracer_record_pair(self, self->pcur_entry->last_line, -first) < 0) {
                     goto error;

coverage/pytracer.py

@@ -168,10 +168,10 @@ def _trace(self, frame, event, arg_unused):
                 # The current opcode is guaranteed to be RESUME. The argument
                 # determines what kind of resume it is.
                 oparg = frame.f_code.co_code[frame.f_lasti + 1]
-                true_call = (oparg == 0)
+                real_call = (oparg == 0)
             else:
-                true_call = (getattr(frame, 'f_lasti', -1) < 0)
-            if true_call:
+                real_call = (getattr(frame, 'f_lasti', -1) < 0)
+            if real_call:
                 self.last_line = -frame.f_code.co_firstlineno
             else:
                 self.last_line = frame.f_lineno
@@ -194,13 +194,22 @@ def _trace(self, frame, event, arg_unused):
                 if RESUME is not None:
                     if len(code) == lasti + 2:
                         # A return from the end of a code object is a real return.
-                        true_return = True
+                        real_return = True
                     else:
                         # it's a real return.
-                        true_return = (code[lasti + 2] != RESUME)
+                        real_return = (code[lasti + 2] != RESUME)
                 else:
-                    true_return = not ( (code[lasti] == YIELD_VALUE) or ((len(code) > lasti + YIELD_FROM_OFFSET) and code[lasti + YIELD_FROM_OFFSET] == YIELD_FROM) )
-                if true_return:
+                    if code[lasti] == RETURN_VALUE:
+                        real_return = True
+                    elif code[lasti] == YIELD_VALUE:
+                        real_return = False
+                    elif len(code) <= lasti + YIELD_FROM_OFFSET:
+                        real_return = True
+                    elif code[lasti + YIELD_FROM_OFFSET] == YIELD_FROM:
+                        real_return = False
+                    else:
+                        real_return = True
+                if real_return:
                     first = frame.f_code.co_firstlineno
                     self.cur_file_data.add((self.last_line, -first))
             # Leaving this function, pop the filename stack.

tests/test_arcs.py

@@ -1288,7 +1288,6 @@ def gen(inp):
             list(gen([1,2,3]))
             """,
             arcz=".1 19 9.  .2 23 34 45 56 63 37 7.",
-            arcz_unpredicted="5.",
         )
 
     def test_abandoned_yield(self):
@@ -1866,7 +1865,6 @@ async def print_sum(x, y):                      # 8
                 ".1 13 38 8E EF FG G. " +
                 "-34 45 56 6-3 " +
                 "-89 9C C-8",
-            arcz_unpredicted="5-3 9-8",
         )
         assert self.stdout() == "Compute 1 + 2 ...\n1 + 2 = 3\n"