diff --git a/mypy-requirements.txt b/mypy-requirements.txt index d356ca0b59d9..229f5624e886 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt==0.1.1 +librt>=0.2.1 diff --git a/mypy/cache.py b/mypy/cache.py index 51deac914efc..f8d3e6a05eba 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -3,8 +3,7 @@ from collections.abc import Sequence from typing import Final -from mypy_extensions import u8 -from native_internal import ( +from librt.internal import ( Buffer as Buffer, read_bool as read_bool, read_float as read_float, @@ -17,6 +16,7 @@ write_str as write_str, write_tag as write_tag, ) +from mypy_extensions import u8 # Always use this type alias to refer to type tags. Tag = u8 diff --git a/mypy/typeshed/stubs/librt/METADATA.toml b/mypy/typeshed/stubs/librt/METADATA.toml index 37dc09b102d4..a42da251bed5 100644 --- a/mypy/typeshed/stubs/librt/METADATA.toml +++ b/mypy/typeshed/stubs/librt/METADATA.toml @@ -1 +1 @@ -version = "0.1.*" +version = "0.2.*" diff --git a/mypy/typeshed/stubs/librt/librt/__init__.pyi b/mypy/typeshed/stubs/librt/librt/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypy/typeshed/stubs/librt/native_internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi similarity index 100% rename from mypy/typeshed/stubs/librt/native_internal.pyi rename to mypy/typeshed/stubs/librt/librt/internal.pyi diff --git a/mypyc/build.py b/mypyc/build.py index efbd0dce31db..40638b31d000 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -42,6 +42,8 @@ from mypyc.namegen import exported_name from mypyc.options import CompilerOptions +LIBRT_MODULES = [("librt.internal", "librt_internal.c")] + try: # Import setuptools so that it monkey-patch overrides distutils import setuptools @@ -492,8 +494,8 @@ def mypycify( strict_dunder_typing: bool = False, group_name: str | None = None, log_trace: bool = False, - depends_on_native_internal: bool = False, - install_native_libs: bool = False, + depends_on_librt_internal: bool = False, + install_librt: bool = False, ) -> list[Extension]: """Main entry point to building using mypyc. @@ -544,11 +546,11 @@ def mypycify( mypyc_trace.txt (derived from executed operations). This is useful for performance analysis, such as analyzing which primitive ops are used the most and on which lines. - depends_on_native_internal: This is True only for mypy itself. - install_native_libs: If True, also build the native extension modules. Normally, - those are build and published on PyPI separately, but during - tests, we want to use their development versions (i.e. from - current commit). + depends_on_librt_internal: This is True only for mypy itself. + install_librt: If True, also build the librt extension modules. Normally, + those are build and published on PyPI separately, but during + tests, we want to use their development versions (i.e. from + current commit). """ # Figure out our configuration @@ -562,7 +564,7 @@ def mypycify( strict_dunder_typing=strict_dunder_typing, group_name=group_name, log_trace=log_trace, - depends_on_native_internal=depends_on_native_internal, + depends_on_librt_internal=depends_on_librt_internal, ) # Generate all the actual important C code @@ -661,21 +663,25 @@ def mypycify( build_single_module(group_sources, cfilenames + shared_cfilenames, cflags) ) - if install_native_libs: - for name in ["native_internal.c"] + RUNTIME_C_FILES: + if install_librt: + os.makedirs("librt", exist_ok=True) + for name in RUNTIME_C_FILES: rt_file = os.path.join(build_dir, name) with open(os.path.join(include_dir(), name), encoding="utf-8") as f: write_file(rt_file, f.read()) - extensions.append( - get_extension()( - "native_internal", - sources=[ - os.path.join(build_dir, file) - for file in ["native_internal.c"] + RUNTIME_C_FILES - ], - include_dirs=[include_dir()], - extra_compile_args=cflags, + for mod, file_name in LIBRT_MODULES: + rt_file = os.path.join(build_dir, file_name) + with open(os.path.join(include_dir(), file_name), encoding="utf-8") as f: + write_file(rt_file, f.read()) + extensions.append( + get_extension()( + mod, + sources=[ + os.path.join(build_dir, file) for file in [file_name] + RUNTIME_C_FILES + ], + include_dirs=[include_dir()], + extra_compile_args=cflags, + ) ) - ) return extensions diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index ca5db52ab7da..3602b3c26e03 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -601,12 +601,12 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: ext_declarations.emit_line(f"#define MYPYC_NATIVE{self.group_suffix}_H") ext_declarations.emit_line("#include ") ext_declarations.emit_line("#include ") - if self.compiler_options.depends_on_native_internal: - ext_declarations.emit_line("#include ") + if self.compiler_options.depends_on_librt_internal: + ext_declarations.emit_line("#include ") declarations = Emitter(self.context) - declarations.emit_line(f"#ifndef MYPYC_NATIVE_INTERNAL{self.group_suffix}_H") - declarations.emit_line(f"#define MYPYC_NATIVE_INTERNAL{self.group_suffix}_H") + declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") + declarations.emit_line(f"#define MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") declarations.emit_line("#include ") declarations.emit_line("#include ") declarations.emit_line(f'#include "__native{self.short_group_suffix}.h"') @@ -1029,8 +1029,8 @@ def emit_module_exec_func( declaration = f"int CPyExec_{exported_name(module_name)}(PyObject *module)" module_static = self.module_internal_static_name(module_name, emitter) emitter.emit_lines(declaration, "{") - if self.compiler_options.depends_on_native_internal: - emitter.emit_line("if (import_native_internal() < 0) {") + if self.compiler_options.depends_on_librt_internal: + emitter.emit_line("if (import_librt_internal() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") emitter.emit_line("PyObject* modname = NULL;") diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 34824a59cd5c..941670ab230d 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -514,7 +514,7 @@ def __hash__(self) -> int: KNOWN_NATIVE_TYPES: Final = { name: RPrimitive(name, is_unboxed=False, is_refcounted=True) - for name in ["native_internal.Buffer"] + for name in ["librt.internal.Buffer"] } diff --git a/mypyc/lib-rt/native_internal.c b/mypyc/lib-rt/librt_internal.c similarity index 96% rename from mypyc/lib-rt/native_internal.c rename to mypyc/lib-rt/librt_internal.c index a6511a1caf25..cb9aa1025821 100644 --- a/mypyc/lib-rt/native_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -2,8 +2,8 @@ #include #include #include "CPy.h" -#define NATIVE_INTERNAL_MODULE -#include "native_internal.h" +#define LIBRT_INTERNAL_MODULE +#include "librt_internal.h" #define START_SIZE 512 #define MAX_SHORT_INT_TAGGED (255 << 1) @@ -558,7 +558,7 @@ write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames return Py_None; } -static PyMethodDef native_internal_module_methods[] = { +static PyMethodDef librt_internal_module_methods[] = { {"write_bool", (PyCFunction)write_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a bool")}, {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")}, {"write_str", (PyCFunction)write_str, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a string")}, @@ -574,11 +574,11 @@ static PyMethodDef native_internal_module_methods[] = { static int NativeInternal_ABI_Version(void) { - return NATIVE_INTERNAL_ABI_VERSION; + return LIBRT_INTERNAL_ABI_VERSION; } static int -native_internal_module_exec(PyObject *m) +librt_internal_module_exec(PyObject *m) { if (PyType_Ready(&BufferType) < 0) { return -1; @@ -604,32 +604,32 @@ native_internal_module_exec(PyObject *m) (void *)read_tag_internal, (void *)NativeInternal_ABI_Version, }; - PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "native_internal._C_API", NULL); + PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { return -1; } return 0; } -static PyModuleDef_Slot native_internal_module_slots[] = { - {Py_mod_exec, native_internal_module_exec}, +static PyModuleDef_Slot librt_internal_module_slots[] = { + {Py_mod_exec, librt_internal_module_exec}, #ifdef Py_MOD_GIL_NOT_USED {Py_mod_gil, Py_MOD_GIL_NOT_USED}, #endif {0, NULL} }; -static PyModuleDef native_internal_module = { +static PyModuleDef librt_internal_module = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = "native_internal", + .m_name = "internal", .m_doc = "Mypy cache serialization utils", .m_size = 0, - .m_methods = native_internal_module_methods, - .m_slots = native_internal_module_slots, + .m_methods = librt_internal_module_methods, + .m_slots = librt_internal_module_slots, }; PyMODINIT_FUNC -PyInit_native_internal(void) +PyInit_internal(void) { - return PyModuleDef_Init(&native_internal_module); + return PyModuleDef_Init(&librt_internal_module); } diff --git a/mypyc/lib-rt/native_internal.h b/mypyc/lib-rt/librt_internal.h similarity index 80% rename from mypyc/lib-rt/native_internal.h rename to mypyc/lib-rt/librt_internal.h index 63e902a6e1bf..fd8ec2422cc5 100644 --- a/mypyc/lib-rt/native_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -1,9 +1,9 @@ -#ifndef NATIVE_INTERNAL_H -#define NATIVE_INTERNAL_H +#ifndef LIBRT_INTERNAL_H +#define LIBRT_INTERNAL_H -#define NATIVE_INTERNAL_ABI_VERSION 0 +#define LIBRT_INTERNAL_ABI_VERSION 0 -#ifdef NATIVE_INTERNAL_MODULE +#ifdef LIBRT_INTERNAL_MODULE static PyObject *Buffer_internal(PyObject *source); static PyObject *Buffer_internal_empty(void); @@ -40,17 +40,21 @@ static void **NativeInternal_API; #define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13]) static int -import_native_internal(void) +import_librt_internal(void) { - NativeInternal_API = (void **)PyCapsule_Import("native_internal._C_API", 0); + PyObject *mod = PyImport_ImportModule("librt.internal"); + if (mod == NULL) + return -1; + Py_DECREF(mod); // we import just for the side effect of making the below work. + NativeInternal_API = (void **)PyCapsule_Import("librt.internal._C_API", 0); if (NativeInternal_API == NULL) return -1; - if (NativeInternal_ABI_Version() != NATIVE_INTERNAL_ABI_VERSION) { - PyErr_SetString(PyExc_ValueError, "ABI version conflict for native_internal"); + if (NativeInternal_ABI_Version() != LIBRT_INTERNAL_ABI_VERSION) { + PyErr_SetString(PyExc_ValueError, "ABI version conflict for librt.internal"); return -1; } return 0; } #endif -#endif // NATIVE_INTERNAL_H +#endif // LIBRT_INTERNAL_H diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 36b55e44dcd1..b78ad0dbc23e 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -75,9 +75,9 @@ def run(self) -> None: setup( ext_modules=[ Extension( - "native_internal", + "librt.internal", [ - "native_internal.c", + "librt_internal.c", "init.c", "int_ops.c", "exc_ops.c", diff --git a/mypyc/options.py b/mypyc/options.py index c009d3c6a7a4..e004a0a52c95 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -17,7 +17,7 @@ def __init__( strict_dunder_typing: bool = False, group_name: str | None = None, log_trace: bool = False, - depends_on_native_internal: bool = False, + depends_on_librt_internal: bool = False, ) -> None: self.strip_asserts = strip_asserts self.multi_file = multi_file @@ -51,7 +51,7 @@ def __init__( # mypyc_trace.txt when compiled module is executed. This is useful for # performance analysis. self.log_trace = log_trace - # If enabled, add capsule imports of native_internal API. This should be used + # If enabled, add capsule imports of librt.internal API. This should be used # only for mypy itself, third-party code compiled with mypyc should not use - # native_internal. - self.depends_on_native_internal = depends_on_native_internal + # librt.internal. + self.depends_on_librt_internal = depends_on_librt_internal diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 8e6e450c64dc..18d475fe89d4 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -333,11 +333,11 @@ error_kind=ERR_NEVER, ) -buffer_rprimitive = KNOWN_NATIVE_TYPES["native_internal.Buffer"] +buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.Buffer"] # Buffer(source) function_op( - name="native_internal.Buffer", + name="librt.internal.Buffer", arg_types=[bytes_rprimitive], return_type=buffer_rprimitive, c_function_name="Buffer_internal", @@ -346,7 +346,7 @@ # Buffer() function_op( - name="native_internal.Buffer", + name="librt.internal.Buffer", arg_types=[], return_type=buffer_rprimitive, c_function_name="Buffer_internal_empty", @@ -362,7 +362,7 @@ ) function_op( - name="native_internal.write_bool", + name="librt.internal.write_bool", arg_types=[object_rprimitive, bool_rprimitive], return_type=none_rprimitive, c_function_name="write_bool_internal", @@ -370,7 +370,7 @@ ) function_op( - name="native_internal.read_bool", + name="librt.internal.read_bool", arg_types=[object_rprimitive], return_type=bool_rprimitive, c_function_name="read_bool_internal", @@ -378,7 +378,7 @@ ) function_op( - name="native_internal.write_str", + name="librt.internal.write_str", arg_types=[object_rprimitive, str_rprimitive], return_type=none_rprimitive, c_function_name="write_str_internal", @@ -386,7 +386,7 @@ ) function_op( - name="native_internal.read_str", + name="librt.internal.read_str", arg_types=[object_rprimitive], return_type=str_rprimitive, c_function_name="read_str_internal", @@ -394,7 +394,7 @@ ) function_op( - name="native_internal.write_float", + name="librt.internal.write_float", arg_types=[object_rprimitive, float_rprimitive], return_type=none_rprimitive, c_function_name="write_float_internal", @@ -402,7 +402,7 @@ ) function_op( - name="native_internal.read_float", + name="librt.internal.read_float", arg_types=[object_rprimitive], return_type=float_rprimitive, c_function_name="read_float_internal", @@ -410,7 +410,7 @@ ) function_op( - name="native_internal.write_int", + name="librt.internal.write_int", arg_types=[object_rprimitive, int_rprimitive], return_type=none_rprimitive, c_function_name="write_int_internal", @@ -418,7 +418,7 @@ ) function_op( - name="native_internal.read_int", + name="librt.internal.read_int", arg_types=[object_rprimitive], return_type=int_rprimitive, c_function_name="read_int_internal", @@ -426,7 +426,7 @@ ) function_op( - name="native_internal.write_tag", + name="librt.internal.write_tag", arg_types=[object_rprimitive, uint8_rprimitive], return_type=none_rprimitive, c_function_name="write_tag_internal", @@ -434,7 +434,7 @@ ) function_op( - name="native_internal.read_tag", + name="librt.internal.read_tag", arg_types=[object_rprimitive], return_type=uint8_rprimitive, c_function_name="read_tag_internal", diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 94e89f276eeb..c4410b3b19d2 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1452,7 +1452,7 @@ class TestOverload: [case testNativeBufferFastPath] from typing import Final from mypy_extensions import u8 -from native_internal import ( +from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag ) @@ -1476,11 +1476,11 @@ def foo() -> None: u = read_tag(b) [out] def foo(): - r0, b :: native_internal.Buffer + r0, b :: librt.internal.Buffer r1 :: str r2, r3, r4, r5, r6 :: None r7 :: bytes - r8 :: native_internal.Buffer + r8 :: librt.internal.Buffer r9, x :: str r10, y :: bool r11, z :: float diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index ab1dcb926c34..8755169fdb0b 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2710,10 +2710,10 @@ from native import Player [out] Player.MIN = -[case testBufferRoundTrip_native_libs] +[case testBufferRoundTrip_librt_internal] from typing import Final from mypy_extensions import u8 -from native_internal import ( +from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag ) diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 172a1016dd91..22ab18e97293 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -86,7 +86,7 @@ setup(name='test_run_output', ext_modules=mypycify({}, separate={}, skip_cgen_input={!r}, strip_asserts=False, - multi_file={}, opt_level='{}', install_native_libs={}), + multi_file={}, opt_level='{}', install_librt={}), ) """ @@ -239,13 +239,16 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> groups = construct_groups(sources, separate, len(module_names) > 1, None) - native_libs = "_native_libs" in testcase.name + # Use _librt_internal to test mypy-specific parts of librt (they have + # some special-casing in mypyc), for everything else use _librt suffix. + librt_internal = testcase.name.endswith("_librt_internal") + librt = librt_internal or testcase.name.endswith("_librt") try: compiler_options = CompilerOptions( multi_file=self.multi_file, separate=self.separate, strict_dunder_typing=self.strict_dunder_typing, - depends_on_native_internal=native_libs, + depends_on_librt_internal=librt_internal, ) result = emitmodule.parse_and_typecheck( sources=sources, @@ -278,7 +281,7 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> with open(setup_file, "w", encoding="utf-8") as f: f.write( setup_format.format( - module_paths, separate, cfiles, self.multi_file, opt_level, native_libs + module_paths, separate, cfiles, self.multi_file, opt_level, librt ) ) diff --git a/pyproject.toml b/pyproject.toml index 1575a15e9909..adcca65bf015 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt==0.1.1", + "librt>=0.2.1", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt==0.1.1", + "librt>=0.2.1", ] dynamic = ["version"] diff --git a/setup.py b/setup.py index 3f53d96dbd85..1d093ec3b9e2 100644 --- a/setup.py +++ b/setup.py @@ -155,7 +155,7 @@ def run(self) -> None: multi_file=sys.platform == "win32" or force_multifile, log_trace=log_trace, # Mypy itself is allowed to use native_internal extension. - depends_on_native_internal=True, + depends_on_librt_internal=True, ) else: diff --git a/test-requirements.txt b/test-requirements.txt index 0983dc362c8a..7e8e167300dc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.13 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.1.1 +librt==0.2.1 # via -r mypy-requirements.txt lxml==6.0.1 ; python_version < "3.15" # via -r test-requirements.in