Python 3.10: support for cython and frame eval mode.

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_modify_bytecode.py

@@ -3,6 +3,7 @@
 from functools import partial
 import itertools
 import os.path
+import sys
 
 from _pydevd_frame_eval.vendored import bytecode
 from _pydevd_frame_eval.vendored.bytecode.instr import Instr, Label
@@ -198,6 +199,36 @@ def __iter__(self):
             node = node.next
 
 
+_PREDICT_TABLE = {
+    'LIST_APPEND': ('JUMP_ABSOLUTE',),
+    'SET_ADD': ('JUMP_ABSOLUTE',),
+    'GET_ANEXT': ('LOAD_CONST',),
+    'GET_AWAITABLE': ('LOAD_CONST',),
+    'DICT_MERGE': ('CALL_FUNCTION_EX',),
+    'MAP_ADD': ('JUMP_ABSOLUTE',),
+    'COMPARE_OP': ('POP_JUMP_IF_FALSE', 'POP_JUMP_IF_TRUE',),
+    'IS_OP': ('POP_JUMP_IF_FALSE', 'POP_JUMP_IF_TRUE',),
+    'CONTAINS_OP': ('POP_JUMP_IF_FALSE', 'POP_JUMP_IF_TRUE',),
+
+    # Note: there are some others with PREDICT on ceval, but they have more logic
+    # and it needs more experimentation to know how it behaves in the static generated
+    # code (and it's only an issue for us if there's actually a line change between
+    # those, so, we don't have to really handle all the cases, only the one where
+    # the line number actually changes from one instruction to the predicted one).
+}
+
+# 3.10 optimizations include copying code branches multiple times (for instance
+# if the body of a finally has a single assign statement it can copy the assign to the case
+# where an exception happens and doesn't happen for optimization purposes) and as such
+# we need to add the programmatic breakpoint multiple times.
+TRACK_MULTIPLE_BRANCHES = sys.version_info[:2] >= (3, 10)
+
+# When tracking multiple branches, we try to fix the bytecodes which would be PREDICTED in the
+# Python eval loop so that we don't have spurious line events that wouldn't usually be issued
+# in the tracing as they're ignored due to the eval prediction (even though they're in the bytecode).
+FIX_PREDICT = sys.version_info[:2] >= (3, 10)
+
+
 def insert_pydevd_breaks(
         code_to_modify,
         breakpoint_lines,
@@ -239,16 +270,49 @@ def insert_pydevd_breaks(
         modified_breakpoint_lines = breakpoint_lines.copy()
 
         curr_node = helper_list.head
+        added_breaks_in_lines = set()
+        last_lineno = None
         while curr_node is not None:
             instruction = curr_node.data
             instruction_lineno = getattr(instruction, 'lineno', None)
+            curr_name = getattr(instruction, 'name', None)
+
+            if FIX_PREDICT:
+                predict_targets = _PREDICT_TABLE.get(curr_name)
+                if predict_targets:
+                    # Odd case: the next instruction may have a line number but it doesn't really
+                    # appear in the tracing due to the PREDICT() in ceval, so, fix the bytecode so
+                    # that it does things the way that ceval actually interprets it.
+                    # See: https://mail.python.org/archives/list/python-dev@python.org/thread/CP2PTFCMTK57KM3M3DLJNWGO66R5RVPB/
+                    next_instruction = curr_node.next.data
+                    next_name = getattr(next_instruction, 'name', None)
+                    if next_name in predict_targets:
+                        next_instruction_lineno = getattr(next_instruction, 'lineno', None)
+                        if next_instruction_lineno:
+                            next_instruction.lineno = None
 
             if instruction_lineno is not None:
+                if TRACK_MULTIPLE_BRANCHES:
+                    if last_lineno is None:
+                        last_lineno = instruction_lineno
+                    else:
+                        if last_lineno == instruction_lineno:
+                            # If the previous is a label, someone may jump into it, so, we need to add
+                            # the break even if it's in the same line.
+                            if curr_node.prev.data.__class__ != Label:
+                                # Skip adding this as the line is still the same.
+                                curr_node = curr_node.next
+                                continue
+                        last_lineno = instruction_lineno
+                else:
+                    if instruction_lineno in added_breaks_in_lines:
+                        curr_node = curr_node.next
+                        continue
 
                 if instruction_lineno in modified_breakpoint_lines:
-                    modified_breakpoint_lines.discard(instruction_lineno)
+                    added_breaks_in_lines.add(instruction_lineno)
                     if curr_node.prev is not None and curr_node.prev.data.__class__ == Label \
-                            and getattr(curr_node.data, 'name', None) == 'POP_TOP':
+                            and curr_name == 'POP_TOP':
 
                         # If we have a SETUP_FINALLY where the target is a POP_TOP, we can't change
                         # the target to be the breakpoint instruction (this can crash the interpreter).

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/vendored/bytecode/__init__.py

@@ -35,9 +35,9 @@
     _ConvertBytecodeToConcrete,
 )
 from _pydevd_frame_eval.vendored.bytecode.cfg import BasicBlock, ControlFlowGraph  # noqa
+import sys
 
-
-def dump_bytecode(bytecode, *, lineno=False):
+def dump_bytecode(bytecode, *, lineno=False, stream=sys.stdout):
     def format_line(index, line):
         nonlocal cur_lineno, prev_lineno
         if lineno:
@@ -87,7 +87,7 @@ def format_instr(instr, labels=None):
             else:
                 fields.append("% 3s    %s" % (offset, format_instr(instr)))
                 line = "".join(fields)
-            print(line)
+            print(line, file=stream)
 
             offset += instr.size
     elif isinstance(bytecode, Bytecode):
@@ -101,30 +101,30 @@ def format_instr(instr, labels=None):
                 label = labels[instr]
                 line = "%s:" % label
                 if index != 0:
-                    print()
+                    print(file=stream)
             else:
                 if instr.lineno is not None:
                     cur_lineno = instr.lineno
                 line = format_instr(instr, labels)
                 line = indent + format_line(index, line)
-            print(line)
-        print()
+            print(line, file=stream)
+        print(file=stream)
     elif isinstance(bytecode, ControlFlowGraph):
         labels = {}
         for block_index, block in enumerate(bytecode, 1):
             labels[id(block)] = "block%s" % block_index
 
         for block_index, block in enumerate(bytecode, 1):
-            print("%s:" % labels[id(block)])
+            print("%s:" % labels[id(block)], file=stream)
             prev_lineno = None
             for index, instr in enumerate(block):
                 if instr.lineno is not None:
                     cur_lineno = instr.lineno
                 line = format_instr(instr, labels)
                 line = indent + format_line(index, line)
-                print(line)
+                print(line, file=stream)
             if block.next_block is not None:
-                print(indent + "-> %s" % labels[id(block.next_block)])
-            print()
+                print(indent + "-> %s" % labels[id(block.next_block)], file=stream)
+            print(file=stream)
     else:
         raise TypeError("unknown bytecode class")

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/vendored/bytecode/concrete.py

@@ -284,11 +284,7 @@ def _assemble_lnotab(first_lineno, linenos):
         return b"".join(lnotab)
 
     @staticmethod
-    def _pack_linetable(doff, dlineno):
-        linetable = []
-        while doff > 254:
-            linetable.append(b"\xfe\x00")
-            doff -= 254
+    def _pack_linetable(doff, dlineno, linetable):
 
         while dlineno < -127:
             linetable.append(struct.pack("Bb", 0, -127))
@@ -298,21 +294,34 @@ def _pack_linetable(doff, dlineno):
             linetable.append(struct.pack("Bb", 0, 127))
             dlineno -= 127
 
+        if doff > 254:
+            linetable.append(struct.pack("Bb", 254, dlineno))
+            doff -= 254
+
+            while doff > 254:
+                linetable.append(b"\xfe\x00")
+                doff -= 254
+            linetable.append(struct.pack("Bb", doff, 0))
+
+        else:
+            linetable.append(struct.pack("Bb", doff, dlineno))
+
         assert 0 <= doff <= 254
         assert -127 <= dlineno <= 127
 
-        linetable.append(struct.pack("Bb", doff, dlineno))
-        return linetable
 
     def _assemble_linestable(self, first_lineno, linenos):
         if not linenos:
             return b""
 
         linetable = []
         old_offset = 0
-        offset, i_size, old_lineno = linenos[0]
+
+        iter_in = iter(linenos)
+
+        offset, i_size, old_lineno = next(iter_in)
         old_dlineno = old_lineno - first_lineno
-        for offset, i_size, lineno in linenos[1:]:
+        for offset, i_size, lineno in iter_in:
             dlineno = lineno - old_lineno
             if dlineno == 0:
                 continue
@@ -321,12 +330,12 @@ def _assemble_linestable(self, first_lineno, linenos):
             doff = offset - old_offset
             old_offset = offset
 
-            linetable.extend(self._pack_linetable(doff, old_dlineno))
+            self._pack_linetable(doff, old_dlineno, linetable)
             old_dlineno = dlineno
 
         # Pack the line of the last instruction.
         doff = offset + i_size - old_offset
-        linetable.extend(self._pack_linetable(doff, old_dlineno))
+        self._pack_linetable(doff, old_dlineno, linetable)
 
         return b"".join(linetable)
 

src/debugpy/_vendored/pydevd/build_tools/build.py

@@ -104,13 +104,13 @@ def build():
         # set VS100COMNTOOLS=C:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools
 
         env = os.environ.copy()
-        if sys.version_info[:2] in ((2, 6), (2, 7), (3, 5), (3, 6), (3, 7), (3, 8), (3, 9)):
+        if sys.version_info[:2] in ((2, 6), (2, 7), (3, 5), (3, 6), (3, 7), (3, 8), (3, 9), (3, 10)):
             import setuptools  # We have to import it first for the compiler to be found
             from distutils import msvc9compiler
 
             if sys.version_info[:2] in ((2, 6), (2, 7)):
                 vcvarsall = msvc9compiler.find_vcvarsall(9.0)
-            elif sys.version_info[:2] in ((3, 5), (3, 6), (3, 7), (3, 8), (3, 9)):
+            elif sys.version_info[:2] in ((3, 5), (3, 6), (3, 7), (3, 8), (3, 9), (3, 10)):
                 vcvarsall = msvc9compiler.find_vcvarsall(14.0)
             if vcvarsall is None or not os.path.exists(vcvarsall):
                 raise RuntimeError('Error finding vcvarsall.')

src/debugpy/_vendored/pydevd/tests_python/resources/_bytecode_constructs.py

@@ -62,13 +62,38 @@ def method8():
 
 
 def method9():
+    # As a note, Python 3.10 is eager to optimize this case and it duplicates the _c = 20
+    # in a codepath where the exception is raised and another where it's not raised.
+    # The frame eval mode must modify the bytecode so that both paths have the
+    # programmatic breakpoint added!
     try:
         _a = 10
     except:
         _b = 10
     finally:_c = 20  # break finally 2
 
 
+def method9a():
+    # Same as method9, but with exception raised (but handled).
+    try:
+        raise AssertionError()
+    except:
+        _b = 10
+    finally:_c = 20  # break finally 3
+
+
+def method9b():
+    # Same as method9, but with exception raised (unhandled).
+    try:
+        try:
+            raise RuntimeError()
+        except AssertionError:
+            _b = 10
+        finally:_c = 20  # break finally 4
+    except:
+        pass
+
+
 def method10():
     _a = {
         0: 0,
@@ -94,6 +119,8 @@ def method11():
     method7()
     method8()
     method9()
+    method9a()
+    method9b()
     method10()
     method11()
     print('TEST SUCEEDED')

src/debugpy/_vendored/pydevd/tests_python/resources/_bytecode_overflow_example.py

@@ -1,3 +1,9 @@
+def check_backtrack(x):  # line 1
+    if not (x == 'a'  # line 2
+        or x == 'c'):  # line 3
+        pass  # line 4
+
+
 import re
 import sys
 

src/debugpy/_vendored/pydevd/tests_python/test_bytecode_manipulation.py

@@ -158,6 +158,12 @@ def test_set_pydevd_break_01():
     check('_bytecode_overflow_example.py', _bytecode_overflow_example.Dummy.fun, method_kwargs={'text': 'ing'})
 
 
+def test_set_pydevd_break_01a():
+    from tests_python.resources import _bytecode_overflow_example
+
+    check('_bytecode_overflow_example.py', _bytecode_overflow_example.check_backtrack, method_kwargs={'x': 'f'})
+
+
 def test_set_pydevd_break_02():
     from tests_python.resources import _bytecode_many_names_example
 

src/debugpy/_vendored/pydevd/tests_python/test_debugger.py

@@ -4263,6 +4263,8 @@ def test_frame_eval_mode_corner_case_04(case_setup):
         'break finally 1',
         'break except 2',
         'break finally 2',
+        'break finally 3',
+        'break finally 4',
         'break in dict',
         'break else',
     ]

src/debugpy/_vendored/pydevd/tests_runfiles/test_runfiles.py

@@ -1,6 +1,7 @@
 import os.path
 import sys
-from tests_python.test_debugger import IS_PY26
+
+IS_PY26 = sys.version_info[:2] == (2, 6)
 
 IS_JYTHON = sys.platform.find('java') != -1