gh-91719: Add pycore_opcode.h internal header file (#91906)

Move the following API from Include/opcode.h (public C API) to a new
Include/internal/pycore_opcode.h header file (internal C API):

* EXTRA_CASES
* _PyOpcode_Caches
* _PyOpcode_Deopt
* _PyOpcode_Jump
* _PyOpcode_OpName
* _PyOpcode_RelativeJump

Makefile.pre.in

@@ -1311,8 +1311,10 @@ regen-opcode:
 	# using Tools/scripts/generate_opcode_h.py
 	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \
 		$(srcdir)/Lib/opcode.py \
-		$(srcdir)/Include/opcode.h.new
+		$(srcdir)/Include/opcode.h.new \
+		$(srcdir)/Include/internal/pycore_opcode.h.new
 	$(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new
+	$(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new
 
 .PHONY: regen-token
 regen-token:

Objects/codeobject.c

@@ -5,6 +5,7 @@
 #include "structmember.h"         // PyMemberDef
 #include "pycore_code.h"          // _PyCodeConstructor
 #include "pycore_interp.h"        // PyInterpreterState.co_extra_freefuncs
+#include "pycore_opcode.h"        // _PyOpcode_Deopt
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_tuple.h"         // _PyTuple_ITEMS()
 #include "clinic/codeobject.c.h"

Objects/frameobject.c

@@ -2,10 +2,11 @@
 
 #include "Python.h"
 #include "pycore_ceval.h"         // _PyEval_BuiltinsFromGlobals()
-#include "pycore_moduleobject.h"  // _PyModule_GetDict()
-#include "pycore_object.h"        // _PyObject_GC_UNTRACK()
 #include "pycore_code.h"          // CO_FAST_LOCAL, etc.
 #include "pycore_function.h"      // _PyFunction_FromConstructor()
+#include "pycore_moduleobject.h"  // _PyModule_GetDict()
+#include "pycore_object.h"        // _PyObject_GC_UNTRACK()
+#include "pycore_opcode.h"        // _PyOpcode_Caches
 
 #include "frameobject.h"          // PyFrameObject
 #include "pycore_frame.h"

Objects/genobject.c

@@ -3,12 +3,12 @@
 #include "Python.h"
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "pycore_ceval.h"         // _PyEval_EvalFrame()
+#include "pycore_frame.h"         // _PyInterpreterFrame
 #include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
+#include "pycore_opcode.h"        // _PyOpcode_Deopt
 #include "pycore_pyerrors.h"      // _PyErr_ClearExcState()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
-#include "pycore_frame.h"         // _PyInterpreterFrame
-#include "frameobject.h"          // PyFrameObject
 #include "structmember.h"         // PyMemberDef
 #include "opcode.h"               // SEND
 

PCbuild/regen.targets

@@ -14,7 +14,7 @@
       <Argument>-C</Argument>
     </_ASTOutputs>
     <_OpcodeSources Include="$(PySourcePath)Tools\scripts\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" />
-    <_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Python\opcode_targets.h" />
+    <_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Include\internal\pycore_opcode.h;$(PySourcePath)Python\opcode_targets.h" />
     <_TokenSources Include="$(PySourcePath)Grammar\Tokens" />
     <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc">
       <Format>rst</Format>
@@ -59,7 +59,7 @@
           Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)"
           DependsOnTargets="FindPythonForBuild">
     <Message Text="Regenerate @(_OpcodeOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
-    <Exec Command="$(PythonForBuild) Tools\scripts\generate_opcode_h.py Lib\opcode.py Include\opcode.h"
+    <Exec Command="$(PythonForBuild) Tools\scripts\generate_opcode_h.py Lib\opcode.py Include\opcode.h Include\internal\pycore_opcode.h"
           WorkingDirectory="$(PySourcePath)" />
     <Exec Command="$(PythonForBuild) Python\makeopcodetargets.py Python\opcode_targets.h"
           WorkingDirectory="$(PySourcePath)" />

Python/ceval.c

@@ -15,6 +15,7 @@
 #include "pycore_long.h"          // _PyLong_GetZero()
 #include "pycore_object.h"        // _PyObject_GC_TRACK()
 #include "pycore_moduleobject.h"  // PyModuleObject
+#include "pycore_opcode.h"        // EXTRA_CASES
 #include "pycore_pyerrors.h"      // _PyErr_Fetch()
 #include "pycore_pylifecycle.h"   // _PyErr_Print()
 #include "pycore_pymem.h"         // _PyMem_IsPtrFreed()

Python/compile.c

@@ -23,17 +23,18 @@
 
 #include <stdbool.h>
 
+// Need _PyOpcode_RelativeJump of pycore_opcode.h
+#define NEED_OPCODE_TABLES
+
 #include "Python.h"
 #include "pycore_ast.h"           // _PyAST_GetDocString()
-#include "pycore_compile.h"       // _PyFuture_FromAST()
 #include "pycore_code.h"          // _PyCode_New()
-#include "pycore_pymem.h"         // _PyMem_IsPtrFreed()
+#include "pycore_compile.h"       // _PyFuture_FromAST()
 #include "pycore_long.h"          // _PyLong_GetZero()
+#include "pycore_opcode.h"        // _PyOpcode_Caches
+#include "pycore_pymem.h"         // _PyMem_IsPtrFreed()
 #include "pycore_symtable.h"      // PySTEntryObject
 
-#define NEED_OPCODE_TABLES
-#include "opcode.h"               // EXTENDED_ARG
-
 
 #define DEFAULT_BLOCK_SIZE 16
 #define DEFAULT_CODE_SIZE 128

Python/specialize.c

@@ -6,7 +6,7 @@
 #include "pycore_long.h"
 #include "pycore_moduleobject.h"
 #include "pycore_object.h"
-#include "opcode.h"
+#include "pycore_opcode.h"        // _PyOpcode_Caches
 #include "structmember.h"         // struct PyMemberDef, T_OFFSET_EX
 
 #include <stdlib.h> // rand()

Tools/scripts/generate_opcode_h.py

@@ -3,12 +3,16 @@
 import sys
 import tokenize
 
-header = """
-/* Auto-generated by Tools/scripts/generate_opcode_h.py from Lib/opcode.py */
+SCRIPT_NAME = "Tools/scripts/generate_opcode_h.py"
+PYTHON_OPCODE = "Lib/opcode.py"
+
+header = f"""
+// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE}
+
 #ifndef Py_OPCODE_H
 #define Py_OPCODE_H
 #ifdef __cplusplus
-extern "C" {
+extern "C" {{
 #endif
 
 
@@ -28,6 +32,29 @@
 #endif /* !Py_OPCODE_H */
 """
 
+internal_header = f"""
+// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE}
+
+#ifndef Py_INTERNAL_OPCODE_H
+#define Py_INTERNAL_OPCODE_H
+#ifdef __cplusplus
+extern "C" {{
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#include "opcode.h"
+""".lstrip()
+
+internal_footer = """
+#ifdef __cplusplus
+}
+#endif
+#endif  // !Py_INTERNAL_OPCODE_H
+"""
+
 DEFINE = "#define {:<38} {:>3}\n"
 
 UINT32_MASK = (1<<32)-1
@@ -43,7 +70,7 @@ def write_int_array_from_ops(name, ops, out):
     assert bits == 0
     out.write(f"}};\n")
 
-def main(opcode_py, outfile='Include/opcode.h'):
+def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/internal/pycore_opcode.h'):
     opcode = {}
     if hasattr(tokenize, 'open'):
         fp = tokenize.open(opcode_py)   # Python 3.2+
@@ -75,8 +102,10 @@ def main(opcode_py, outfile='Include/opcode.h'):
     opname_including_specialized[255] = 'DO_TRACING'
     used[255] = True
 
-    with open(outfile, 'w') as fobj:
+    with (open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj):
         fobj.write(header)
+        iobj.write(internal_header)
+
         for name in opname:
             if name in opmap:
                 fobj.write(DEFINE.format(name, opmap[name]))
@@ -86,28 +115,29 @@ def main(opcode_py, outfile='Include/opcode.h'):
         for name, op in specialized_opmap.items():
             fobj.write(DEFINE.format(name, op))
 
-        fobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
-        fobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n")
-        fobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
-        write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj)
-        write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj)
+        iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
+        iobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n")
+        iobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
+        write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], iobj)
+        write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], iobj)
 
-        fobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n")
+        iobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n")
         for i, entries in enumerate(opcode["_inline_cache_entries"]):
             if entries:
-                fobj.write(f"    [{opname[i]}] = {entries},\n")
-        fobj.write("};\n")
+                iobj.write(f"    [{opname[i]}] = {entries},\n")
+        iobj.write("};\n")
+
         deoptcodes = {}
         for basic in opmap:
             deoptcodes[basic] = basic
         for basic, family in opcode["_specializations"].items():
             for specialized in family:
                 deoptcodes[specialized] = basic
-        fobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n")
+        iobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n")
         for opt, deopt in sorted(deoptcodes.items()):
-            fobj.write(f"    [{opt}] = {deopt},\n")
-        fobj.write("};\n")
-        fobj.write("#endif /* OPCODE_TABLES */\n")
+            iobj.write(f"    [{opt}] = {deopt},\n")
+        iobj.write("};\n")
+        iobj.write("#endif   // NEED_OPCODE_TABLES\n")
 
         fobj.write("\n")
         fobj.write("#define HAS_CONST(op) (false\\")
@@ -119,28 +149,29 @@ def main(opcode_py, outfile='Include/opcode.h'):
         for i, (op, _) in enumerate(opcode["_nb_ops"]):
             fobj.write(DEFINE.format(op, i))
 
-        fobj.write("\n")
-        fobj.write("#ifdef Py_DEBUG\n")
-        fobj.write("static const char *const _PyOpcode_OpName[256] = {\n")
+        iobj.write("\n")
+        iobj.write("#ifdef Py_DEBUG\n")
+        iobj.write("static const char *const _PyOpcode_OpName[256] = {\n")
         for op, name in enumerate(opname_including_specialized):
             if name[0] != "<":
                 op = name
-            fobj.write(f'''    [{op}] = "{name}",\n''')
-        fobj.write("};\n")
-        fobj.write("#endif\n")
+            iobj.write(f'''    [{op}] = "{name}",\n''')
+        iobj.write("};\n")
+        iobj.write("#endif\n")
 
-        fobj.write("\n")
-        fobj.write("#define EXTRA_CASES \\\n")
+        iobj.write("\n")
+        iobj.write("#define EXTRA_CASES \\\n")
         for i, flag in enumerate(used):
             if not flag:
-                fobj.write(f"    case {i}: \\\n")
-        fobj.write("        ;\n")
+                iobj.write(f"    case {i}: \\\n")
+        iobj.write("        ;\n")
 
         fobj.write(footer)
+        iobj.write(internal_footer)
 
 
     print(f"{outfile} regenerated from {opcode_py}")
 
 
 if __name__ == '__main__':
-    main(sys.argv[1], sys.argv[2])
+    main(sys.argv[1], sys.argv[2], sys.argv[3])