gh-105481: Generate the opcode lists in dis from data extracted from …

…bytecodes.c (#106758)

.gitattributes

@@ -87,7 +87,6 @@ Programs/test_frozenmain.h                          generated
 Python/Python-ast.c                                 generated
 Python/executor_cases.c.h                           generated
 Python/generated_cases.c.h                          generated
-Include/internal/pycore_opcode_metadata.h           generated
 Python/opcode_targets.h                             generated
 Python/stdlib_module_names.h                        generated
 Tools/peg_generator/pegen/grammar_parser.py         generated

Doc/library/dis.rst

@@ -1803,15 +1803,12 @@ instructions:
    Sequence of bytecodes that access an attribute by name.
 
 
-.. data:: hasjrel
-
-   Sequence of bytecodes that have a relative jump target.
+.. data:: hasjump
 
+   Sequence of bytecodes that have a jump target. All jumps
+   are relative.
 
-.. data:: hasjabs
-
-   Sequence of bytecodes that have an absolute jump target.
-
+   .. versionadded:: 3.13
 
 .. data:: haslocal
 
@@ -1827,3 +1824,20 @@ instructions:
    Sequence of bytecodes that set an exception handler.
 
    .. versionadded:: 3.12
+
+
+.. data:: hasjrel
+
+   Sequence of bytecodes that have a relative jump target.
+
+   .. deprecated:: 3.13
+      All jumps are now relative. Use :data:`hasjump`.
+
+
+.. data:: hasjabs
+
+   Sequence of bytecodes that have an absolute jump target.
+
+   .. deprecated:: 3.13
+      All jumps are now relative. This list is empty.
+

Include/cpython/compile.h

@@ -73,3 +73,7 @@ PyAPI_FUNC(int) PyUnstable_OpcodeHasArg(int opcode);
 PyAPI_FUNC(int) PyUnstable_OpcodeHasConst(int opcode);
 PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode);
 PyAPI_FUNC(int) PyUnstable_OpcodeHasJump(int opcode);
+PyAPI_FUNC(int) PyUnstable_OpcodeHasFree(int opcode);
+PyAPI_FUNC(int) PyUnstable_OpcodeHasLocal(int opcode);
+PyAPI_FUNC(int) PyUnstable_OpcodeHasExc(int opcode);
+

Include/internal/pycore_opcode_metadata.h

@@ -955,10 +955,14 @@ enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT
 #define HAS_CONST_FLAG (2)
 #define HAS_NAME_FLAG (4)
 #define HAS_JUMP_FLAG (8)
+#define HAS_FREE_FLAG (16)
+#define HAS_LOCAL_FLAG (32)
 #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG))
 #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG))
 #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG))
 #define OPCODE_HAS_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_JUMP_FLAG))
+#define OPCODE_HAS_FREE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_FREE_FLAG))
+#define OPCODE_HAS_LOCAL(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_LOCAL_FLAG))
 
 struct opcode_metadata {
     bool valid_entry;
@@ -995,16 +999,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [NOP] = { true, INSTR_FMT_IX, 0 },
     [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
     [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
+    [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
     [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG },
-    [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [STORE_FAST_MAYBE_NULL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
+    [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [STORE_FAST_MAYBE_NULL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
     [POP_TOP] = { true, INSTR_FMT_IX, 0 },
     [PUSH_NULL] = { true, INSTR_FMT_IX, 0 },
     [END_FOR] = { true, INSTR_FMT_IB, 0 },
@@ -1028,7 +1032,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IBC, 0 },
     [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IBC, 0 },
     [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IBC, 0 },
-    [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, 0 },
+    [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, HAS_LOCAL_FLAG },
     [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, 0 },
     [BINARY_SLICE] = { true, INSTR_FMT_IX, 0 },
     [STORE_SLICE] = { true, INSTR_FMT_IX, 0 },
@@ -1080,12 +1084,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG },
     [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG },
     [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG },
-    [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
+    [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG },
+    [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG },
+    [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG },
+    [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG },
+    [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG },
     [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
     [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
     [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },

Lib/opcode.py

@@ -4,10 +4,12 @@
 operate on bytecodes (e.g. peephole optimizers).
 """
 
-__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs",
-           "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap",
-           "stack_effect", "HAVE_ARGUMENT", "EXTENDED_ARG"]
 
+# Note that __all__ is further extended below
+__all__ = ["cmp_op", "opname", "opmap", "stack_effect", "hascompare",
+           "HAVE_ARGUMENT", "EXTENDED_ARG"]
+
+import _opcode
 from _opcode import stack_effect
 
 import sys
@@ -17,55 +19,24 @@
 
 cmp_op = ('<', '<=', '==', '!=', '>', '>=')
 
-hasarg = []
-hasconst = []
-hasname = []
-hasjrel = []
-hasjabs = []
-haslocal = []
-hascompare = []
-hasfree = []
-hasexc = []
-
 
 ENABLE_SPECIALIZATION = True
 
 def is_pseudo(op):
     return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE
 
-oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs,
-           haslocal, hascompare, hasfree, hasexc]
-
 opmap = {}
 
-## pseudo opcodes (used in the compiler) mapped to the values
-## they can become in the actual code.
+# pseudo opcodes (used in the compiler) mapped to the values
+# they can become in the actual code.
 _pseudo_ops = {}
 
 def def_op(name, op):
     opmap[name] = op
 
-def name_op(name, op):
-    def_op(name, op)
-    hasname.append(op)
-
-def jrel_op(name, op):
-    def_op(name, op)
-    hasjrel.append(op)
-
-def jabs_op(name, op):
-    def_op(name, op)
-    hasjabs.append(op)
-
 def pseudo_op(name, op, real_ops):
     def_op(name, op)
     _pseudo_ops[name] = real_ops
-    # add the pseudo opcode to the lists its targets are in
-    for oplist in oplists:
-        res = [opmap[rop] in oplist for rop in real_ops]
-        if any(res):
-            assert all(res)
-            oplist.append(op)
 
 
 # Instruction opcodes for compiled code
@@ -137,74 +108,61 @@ def pseudo_op(name, op, real_ops):
 
 HAVE_ARGUMENT = 90             # real opcodes from here have an argument:
 
-name_op('STORE_NAME', 90)       # Index in name list
-name_op('DELETE_NAME', 91)      # ""
+def_op('STORE_NAME', 90)       # Index in name list
+def_op('DELETE_NAME', 91)      # ""
 def_op('UNPACK_SEQUENCE', 92)   # Number of tuple items
-jrel_op('FOR_ITER', 93)
+def_op('FOR_ITER', 93)
 def_op('UNPACK_EX', 94)
-name_op('STORE_ATTR', 95)       # Index in name list
-name_op('DELETE_ATTR', 96)      # ""
-name_op('STORE_GLOBAL', 97)     # ""
-name_op('DELETE_GLOBAL', 98)    # ""
+def_op('STORE_ATTR', 95)       # Index in name list
+def_op('DELETE_ATTR', 96)      # ""
+def_op('STORE_GLOBAL', 97)     # ""
+def_op('DELETE_GLOBAL', 98)    # ""
 def_op('SWAP', 99)
 def_op('LOAD_CONST', 100)       # Index in const list
-hasconst.append(100)
-name_op('LOAD_NAME', 101)       # Index in name list
+def_op('LOAD_NAME', 101)       # Index in name list
 def_op('BUILD_TUPLE', 102)      # Number of tuple items
 def_op('BUILD_LIST', 103)       # Number of list items
 def_op('BUILD_SET', 104)        # Number of set items
 def_op('BUILD_MAP', 105)        # Number of dict entries
-name_op('LOAD_ATTR', 106)       # Index in name list
+def_op('LOAD_ATTR', 106)       # Index in name list
 def_op('COMPARE_OP', 107)       # Comparison operator
-hascompare.append(107)
-name_op('IMPORT_NAME', 108)     # Index in name list
-name_op('IMPORT_FROM', 109)     # Index in name list
-jrel_op('JUMP_FORWARD', 110)    # Number of words to skip
-
-jrel_op('POP_JUMP_IF_FALSE', 114)
-jrel_op('POP_JUMP_IF_TRUE', 115)
-name_op('LOAD_GLOBAL', 116)     # Index in name list
+def_op('IMPORT_NAME', 108)     # Index in name list
+def_op('IMPORT_FROM', 109)     # Index in name list
+def_op('JUMP_FORWARD', 110)    # Number of words to skip
+
+def_op('POP_JUMP_IF_FALSE', 114)
+def_op('POP_JUMP_IF_TRUE', 115)
+def_op('LOAD_GLOBAL', 116)     # Index in name list
 def_op('IS_OP', 117)
 def_op('CONTAINS_OP', 118)
 def_op('RERAISE', 119)
 def_op('COPY', 120)
 def_op('RETURN_CONST', 121)
-hasconst.append(121)
 def_op('BINARY_OP', 122)
-jrel_op('SEND', 123)            # Number of words to skip
+def_op('SEND', 123)            # Number of words to skip
 def_op('LOAD_FAST', 124)        # Local variable number, no null check
-haslocal.append(124)
 def_op('STORE_FAST', 125)       # Local variable number
-haslocal.append(125)
 def_op('DELETE_FAST', 126)      # Local variable number
-haslocal.append(126)
 def_op('LOAD_FAST_CHECK', 127)  # Local variable number
-haslocal.append(127)
-jrel_op('POP_JUMP_IF_NOT_NONE', 128)
-jrel_op('POP_JUMP_IF_NONE', 129)
+def_op('POP_JUMP_IF_NOT_NONE', 128)
+def_op('POP_JUMP_IF_NONE', 129)
 def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
 def_op('GET_AWAITABLE', 131)
 def_op('BUILD_SLICE', 133)      # Number of items
-jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards)
+def_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards)
 def_op('MAKE_CELL', 135)
-hasfree.append(135)
 def_op('LOAD_DEREF', 137)
-hasfree.append(137)
 def_op('STORE_DEREF', 138)
-hasfree.append(138)
 def_op('DELETE_DEREF', 139)
-hasfree.append(139)
-jrel_op('JUMP_BACKWARD', 140)    # Number of words to skip (backwards)
-name_op('LOAD_SUPER_ATTR', 141)
+def_op('JUMP_BACKWARD', 140)    # Number of words to skip (backwards)
+def_op('LOAD_SUPER_ATTR', 141)
 def_op('CALL_FUNCTION_EX', 142)  # Flags
 def_op('LOAD_FAST_AND_CLEAR', 143)  # Local variable number
-haslocal.append(143)
 def_op('EXTENDED_ARG', 144)
-EXTENDED_ARG = 144
+EXTENDED_ARG = opmap['EXTENDED_ARG']
 def_op('LIST_APPEND', 145)
 def_op('SET_ADD', 146)
 def_op('MAP_ADD', 147)
-hasfree.append(148)
 def_op('COPY_FREE_VARS', 149)
 def_op('YIELD_VALUE', 150)
 def_op('RESUME', 151)   # This must be kept in sync with deepfreeze.py
@@ -224,12 +182,10 @@ def pseudo_op(name, op, real_ops):
 def_op('STORE_FAST_STORE_FAST', 170)
 def_op('CALL', 171)
 def_op('KW_NAMES', 172)
-hasconst.append(172)
 def_op('CALL_INTRINSIC_1', 173)
 def_op('CALL_INTRINSIC_2', 174)
-name_op('LOAD_FROM_DICT_OR_GLOBALS', 175)
+def_op('LOAD_FROM_DICT_OR_GLOBALS', 175)
 def_op('LOAD_FROM_DICT_OR_DEREF', 176)
-hasfree.append(176)
 def_op('SET_FUNCTION_ATTRIBUTE', 177)    # Attribute
 
 # Optimizer hook
@@ -258,16 +214,12 @@ def pseudo_op(name, op, real_ops):
 def_op('INSTRUMENTED_LINE', 254)
 # 255 is reserved
 
-hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT])
 
 MIN_PSEUDO_OPCODE = 256
 
 pseudo_op('SETUP_FINALLY', 256, ['NOP'])
-hasexc.append(256)
 pseudo_op('SETUP_CLEANUP', 257, ['NOP'])
-hasexc.append(257)
 pseudo_op('SETUP_WITH', 258, ['NOP'])
-hasexc.append(258)
 pseudo_op('POP_BLOCK', 259, ['NOP'])
 
 pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD'])
@@ -283,12 +235,29 @@ def pseudo_op(name, op, real_ops):
 
 MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1
 
-del def_op, name_op, jrel_op, jabs_op, pseudo_op
+del def_op, pseudo_op
 
 opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)]
 for op, i in opmap.items():
     opname[i] = op
 
+# The build uses older versions of Python which do not have _opcode.has_* functions
+if sys.version_info[:2] >= (3, 13):
+    # These lists are documented as part of the dis module's API
+    hasarg = [op for op in opmap.values() if _opcode.has_arg(op)]
+    hasconst = [op for op in opmap.values() if _opcode.has_const(op)]
+    hasname = [op for op in opmap.values() if _opcode.has_name(op)]
+    hasjump = [op for op in opmap.values() if _opcode.has_jump(op)]
+    hasjrel = hasjump  # for backward compatibility
+    hasjabs = []
+    hasfree = [op for op in opmap.values() if _opcode.has_free(op)]
+    haslocal = [op for op in opmap.values() if _opcode.has_local(op)]
+    hasexc = [op for op in opmap.values() if _opcode.has_exc(op)]
+
+    __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel",
+                    "hasjabs", "hasfree", "haslocal", "hasexc"])
+
+hascompare = [opmap["COMPARE_OP"]]
 
 _nb_ops = [
     ("NB_ADD", "+"),