diff --git a/Cargo.toml b/Cargo.toml index 4c5a083060d..40491735ee3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ experimental-async = ["macros", "pyo3-macros/experimental-async"] # Enables pyo3::inspect module and additional type information on FromPyObject # and IntoPy traits -experimental-inspect = [] +experimental-inspect = ["pyo3-macros/experimental-inspect"] # Enables annotating Rust inline modules with #[pymodule] to build Python modules declaratively experimental-declarative-modules = ["pyo3-macros/experimental-declarative-modules", "macros"] @@ -140,6 +140,7 @@ members = [ "pyo3-build-config", "pyo3-macros", "pyo3-macros-backend", + "pyo3-introspection", "pytests", "examples", ] diff --git a/noxfile.py b/noxfile.py index 84676b1ff0c..f56837a0ccb 100644 --- a/noxfile.py +++ b/noxfile.py @@ -732,6 +732,19 @@ def update_ui_tests(session: nox.Session): _run_cargo(session, *command, "--features=abi3,full", env=env) +@nox.session(name="test-introspection") +def test_introspection(session: nox.Session): + session.run_always("python", "-m", "pip", "install", "-v", "./pytests") + # We look for the built library + lib_file = None + for file in Path(session.virtualenv.location).rglob("pyo3_pytests.*"): + if file.is_file(): + lib_file = str(file.resolve()) + _run_cargo_test( + session, package="pyo3-introspection", env={"PYO3_PYTEST_LIB_PATH": lib_file} + ) + + def _build_docs_for_ffi_check(session: nox.Session) -> None: # pyo3-ffi-check needs to scrape docs of pyo3-ffi _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") @@ -847,6 +860,7 @@ def _run_cargo_test( *, package: Optional[str] = None, features: Optional[str] = None, + env: Optional[Dict[str, str]] = None, ) -> None: command = ["cargo"] if "careful" in session.posargs: @@ -859,7 +873,7 @@ def _run_cargo_test( if features: command.append(f"--features={features}") - _run(session, *command, external=True) + _run(session, *command, external=True, env=env or {}) def _run_cargo_publish(session: nox.Session, *, package: str) -> None: diff --git a/pyo3-introspection/Cargo.toml b/pyo3-introspection/Cargo.toml new file mode 100644 index 00000000000..ef863056b24 --- /dev/null +++ b/pyo3-introspection/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "pyo3-introspection" +version = "0.21.0-beta.0" +description = "Introspect dynamic libraries built with PyO3 to get metdata about the exported Python types" +authors = ["PyO3 Project and Contributors "] +homepage = "https://github.com/pyo3/pyo3" +repository = "https://github.com/pyo3/pyo3" +license = "MIT OR Apache-2.0" +edition = "2021" + +[dependencies] +anyhow = "1" +goblin = "0.8.0" +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[lints] +workspace = true diff --git a/pyo3-introspection/LICENSE-APACHE b/pyo3-introspection/LICENSE-APACHE new file mode 100644 index 00000000000..fca31990733 --- /dev/null +++ b/pyo3-introspection/LICENSE-APACHE @@ -0,0 +1,189 @@ + Copyright (c) 2017-present PyO3 Project and Contributors. https://github.com/PyO3 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/pyo3-introspection/LICENSE-MIT b/pyo3-introspection/LICENSE-MIT new file mode 100644 index 00000000000..cd0ad009a8b --- /dev/null +++ b/pyo3-introspection/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2023-present PyO3 Project and Contributors. https://github.com/PyO3 + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/pyo3-introspection/src/introspection.rs b/pyo3-introspection/src/introspection.rs new file mode 100644 index 00000000000..4c79fd31cfc --- /dev/null +++ b/pyo3-introspection/src/introspection.rs @@ -0,0 +1,140 @@ +use crate::model::{Class, Function, Module}; +use anyhow::{bail, Context, Result}; +use goblin::mach::Mach; +use goblin::Object; +use serde::Deserialize; +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +/// Introspect a cdylib built with PyO3 and returns the definition of a Python module +pub fn introspect_cdylib(library_path: impl AsRef, main_module_name: &str) -> Result { + let chunks = find_introspection_chunks_in_binary_object(library_path.as_ref())?; + parse_chunks(&chunks, main_module_name) +} + +/// Parses the introspection chunks found in the binary +fn parse_chunks(chunks: &[Chunk], main_module_name: &str) -> Result { + let chunks_by_id = chunks + .iter() + .map(|c| { + ( + match c { + Chunk::Module { id, .. } => id, + Chunk::Class { id, .. } => id, + Chunk::Function { id, .. } => id, + }, + c, + ) + }) + .collect::>(); + // We look for the root chunk + for chunk in chunks { + if let Chunk::Module { + name, + members, + id: _, + } = chunk + { + if name == main_module_name { + return parse_module(name, members, &chunks_by_id); + } + } + } + bail!("No module named {main_module_name} found") +} + +fn parse_module( + name: &str, + members: &[String], + chunks_by_id: &HashMap<&String, &Chunk>, +) -> Result { + let mut modules = Vec::new(); + let mut classes = Vec::new(); + let mut functions = Vec::new(); + for member in members { + if let Some(chunk) = chunks_by_id.get(member) { + match chunk { + Chunk::Module { + name, + members, + id: _, + } => { + modules.push(parse_module(name, members, chunks_by_id)?); + } + Chunk::Class { name, id: _ } => classes.push(Class { name: name.into() }), + Chunk::Function { name, id: _ } => functions.push(Function { name: name.into() }), + } + } + } + Ok(Module { + name: name.into(), + modules, + classes, + functions, + }) +} + +fn find_introspection_chunks_in_binary_object(path: &Path) -> Result> { + let library_content = + fs::read(path).with_context(|| format!("Failed to read {}", path.display()))?; + let mut chunks = Vec::new(); + match Object::parse(&library_content) + .context("The built library is not valid or not supported by our binary parser")? + { + Object::Mach(Mach::Binary(matcho)) => { + if !matcho.is_64 { + bail!("Only 64 bits binaries are supported"); + } + if !matcho.little_endian { + bail!("Only little endian binaries are supported"); + } + let Some(text_segment) = matcho + .segments + .iter() + .find(|s| s.segname == *b"__TEXT\0\0\0\0\0\0\0\0\0\0") + else { + bail!("No __TEXT segment found"); + }; + for (sec, sec_content) in text_segment.sections()? { + println!( + "{} {}", + String::from_utf8_lossy(&sec.sectname), + sec_content.len() + ); + } + let Some((_, pyo3_data_section)) = text_segment + .sections()? + .into_iter() + .find(|s| s.0.sectname == *b"__pyo3_data0\0\0\0\0") + else { + bail!("No __pyo3_data0 section found"); + }; + for element in pyo3_data_section.chunks(16) { + let ptr = usize::from_le_bytes(element[..8].try_into().unwrap()); + let len = usize::from_le_bytes(element[8..].try_into().unwrap()); + chunks.push(serde_json::from_slice(&library_content[ptr..ptr + len])?); + } + } + _ => bail!("Only Match-O files can be introspected"), + }; + Ok(chunks) +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum Chunk { + Module { + id: String, + name: String, + members: Vec, + }, + Class { + id: String, + name: String, + }, + Function { + id: String, + name: String, + }, +} diff --git a/pyo3-introspection/src/lib.rs b/pyo3-introspection/src/lib.rs new file mode 100644 index 00000000000..4958bfc0878 --- /dev/null +++ b/pyo3-introspection/src/lib.rs @@ -0,0 +1,4 @@ +pub use crate::introspection::introspect_cdylib; + +mod introspection; +pub mod model; diff --git a/pyo3-introspection/src/model.rs b/pyo3-introspection/src/model.rs new file mode 100644 index 00000000000..73a4c27d082 --- /dev/null +++ b/pyo3-introspection/src/model.rs @@ -0,0 +1,17 @@ +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Module { + pub name: String, + pub modules: Vec, + pub classes: Vec, + pub functions: Vec, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Class { + pub name: String, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Function { + pub name: String, +} diff --git a/pyo3-introspection/tests/test.rs b/pyo3-introspection/tests/test.rs new file mode 100644 index 00000000000..0d5832d511d --- /dev/null +++ b/pyo3-introspection/tests/test.rs @@ -0,0 +1,50 @@ +use anyhow::Result; +use pyo3_introspection::introspect_cdylib; +use pyo3_introspection::model::{Class, Module}; +use std::env; + +#[test] +fn introspect_pytests() -> Result<()> { + let binary = env::var_os("PYO3_PYTEST_LIB_PATH") + .expect("The PYO3_PYTEST_LIB_PATH constant must be set and target the pyo3-pytests cdylib"); + let module = introspect_cdylib(binary, "pyo3_pytests")?; + assert_eq!( + module, + Module { + name: "pyo3_pytests".into(), + modules: vec![ + Module { + name: "pyclasses".into(), + modules: vec![], + classes: vec![ + Class { + name: "AssertingBaseClass".into() + }, + Class { + name: "AssertingBaseClassGilRef".into() + }, + Class { + name: "ClassWithoutConstructor".into() + }, + Class { + name: "EmptyClass".into() + }, + Class { + name: "PyClassIter".into() + } + ], + functions: vec![], + }, + Module { + name: "pyfunctions".into(), + modules: vec![], + classes: vec![], + functions: vec![], + } + ], + classes: vec![], + functions: vec![], + } + ); + Ok(()) +} diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index c2ffd53b0fc..350d2db363a 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -29,3 +29,4 @@ workspace = true [features] experimental-async = [] +experimental-inspect = [] diff --git a/pyo3-macros-backend/src/introspection.rs b/pyo3-macros-backend/src/introspection.rs new file mode 100644 index 00000000000..329df32f5e0 --- /dev/null +++ b/pyo3-macros-backend/src/introspection.rs @@ -0,0 +1,115 @@ +//! Generates introspection data i.e. JSON strings in the .pyo3_data0 section +//! +//! There is a JSON per PyO3 proc macro (pyclass, pymodule, pyfunction...) +//! +//! These JSON blobs can refer to each others via the _PYO3_INTROSPECTION_ID constants +//! providing unique ids for each element. + +use crate::utils::PyO3CratePath; +use proc_macro2::TokenStream; +use quote::{quote, ToTokens}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use syn::Ident; + +static GLOBAL_COUNTER_FOR_UNIQUE_NAMES: AtomicUsize = AtomicUsize::new(0); + +pub fn module_introspection_code<'a>( + pyo3_crate_path: &PyO3CratePath, + name: &str, + members: impl IntoIterator, +) -> TokenStream { + let mut to_concat = Vec::new(); + to_concat.push(quote! { "{\"type\":\"module\",\"id\":\"" }); + to_concat.push(quote! { _PYO3_INTROSPECTION_ID }); + to_concat.push(quote! { "\",\"name\":\""}); + to_concat.push(quote! { #name }); + to_concat.push(quote! { "\",\"members\":["}); + let mut start = true; + for member in members { + if start { + start = false; + } else { + to_concat.push(quote! { "," }); + } + to_concat.push(quote! { "\"" }); + to_concat.push(quote! { + #member::_PYO3_INTROSPECTION_ID + }); + to_concat.push(quote! { "\"" }); + } + to_concat.push(quote! { "]}" }); + let stub = stub_section(quote! { + #pyo3_crate_path::impl_::concat::const_concat!(#(#to_concat , )*) + }); + let introspection_id = introspection_id_const(); + quote! { + #stub + #introspection_id + } +} + +pub fn class_introspection_code( + pyo3_crate_path: &PyO3CratePath, + ident: &Ident, + name: &str, +) -> TokenStream { + let mut to_concat = Vec::new(); + to_concat.push(quote! { "{\"type\":\"class\",\"id\":\"" }); + to_concat.push(quote! { #ident::_PYO3_INTROSPECTION_ID }); + to_concat.push(quote! { "\",\"name\":\""}); + to_concat.push(quote! { #name }); + to_concat.push(quote! { "\"}" }); + let stub = stub_section(quote! { + #pyo3_crate_path::impl_::concat::const_concat!(#(#to_concat , )*) + }); + let introspection_id = introspection_id_const(); + quote! { + #stub + impl #ident { + #introspection_id + } + } +} + +pub fn function_introspection_code(pyo3_crate_path: &PyO3CratePath, name: &str) -> TokenStream { + let mut to_concat = Vec::new(); + to_concat.push(quote! { "{\"type\":\"function\",\"id\":\"" }); + to_concat.push(quote! { _PYO3_INTROSPECTION_ID }); + to_concat.push(quote! { "\",\"name\":\""}); + to_concat.push(quote! { #name }); + to_concat.push(quote! { "\"}" }); + let stub = stub_section(quote! { + #pyo3_crate_path::impl_::concat::const_concat!(#(#to_concat , )*) + }); + let introspection_id = introspection_id_const(); + quote! { + #stub + #introspection_id + } +} + +fn stub_section(content: impl ToTokens) -> TokenStream { + let section_name = if cfg!(any(target_os = "macos", target_os = "ios")) { + "__TEXT,__pyo3_data0" + } else { + ".pyo3_data0" + }; + quote! { + const _: () = { + #[used] + #[link_section = #section_name] + static PYO3_INTROSPECTION_DATA: &'static str = #content; + }; + } +} + +fn introspection_id_const() -> TokenStream { + let id = GLOBAL_COUNTER_FOR_UNIQUE_NAMES.fetch_add(1, Ordering::Relaxed); + quote! { + pub const _PYO3_INTROSPECTION_ID: &'static str = concat!( + env!("CARGO_CRATE_NAME"), + env!("CARGO_PKG_VERSION"), + stringify!(#id) + ); + } +} diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index a9d75a2a6fe..6880465dc88 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -11,6 +11,8 @@ mod utils; mod attributes; mod deprecations; mod frompyobject; +#[cfg(feature = "experimental-inspect")] +mod introspection; mod konst; mod method; mod module; diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 626cde121e6..5d92f3b4a30 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -1,5 +1,7 @@ //! Code generation for the function that initializes a python module and adds classes and function. +#[cfg(feature = "experimental-inspect")] +use crate::introspection::module_introspection_code; use crate::utils::Ctx; use crate::{ attributes::{self, take_attributes, take_pyo3_options, CrateAttribute, NameAttribute}, @@ -77,6 +79,8 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { let ctx = &Ctx::new(&options.krate); let Ctx { pyo3_path } = ctx; let doc = get_doc(attrs, None); + #[cfg(feature = "experimental-inspect")] + let name = options.name.clone().unwrap_or_else(|| ident.unraw()); let mut module_items = Vec::new(); let mut module_items_cfg_attrs = Vec::new(); @@ -242,12 +246,18 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { } } - let initialization = module_initialization(options, ident); + let initialization = module_initialization(&options, ident); + #[cfg(feature = "experimental-inspect")] + let introspection = module_introspection_code(pyo3_path, &name.to_string(), &module_items); + #[cfg(not(feature = "experimental-inspect"))] + let introspection = quote! {}; + Ok(quote!( #vis mod #ident { #(#items)* #initialization + #introspection #[allow(unknown_lints, non_local_definitions)] impl MakeDef { @@ -288,8 +298,15 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result let ident = &function.sig.ident; let vis = &function.vis; let doc = get_doc(&function.attrs, None); + #[cfg(feature = "experimental-inspect")] + let name = options.name.clone().unwrap_or_else(|| ident.unraw()); + + let initialization = module_initialization(&options, ident); - let initialization = module_initialization(options, ident); + #[cfg(feature = "experimental-inspect")] + let introspection = module_introspection_code(pyo3_path, &name.to_string(), &[]); + #[cfg(not(feature = "experimental-inspect"))] + let introspection = quote! {}; // Module function called with optional Python<'_> marker as first arg, followed by the module. let mut module_args = Vec::new(); @@ -328,6 +345,7 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result #[doc(hidden)] #vis mod #ident { #initialization + #introspection } // Generate the definition inside an anonymous function in the same scope as the original function - @@ -354,8 +372,8 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result }) } -fn module_initialization(options: PyModuleOptions, ident: &syn::Ident) -> TokenStream { - let name = options.name.unwrap_or_else(|| ident.unraw()); +fn module_initialization(options: &PyModuleOptions, ident: &syn::Ident) -> TokenStream { + let name = options.name.clone().unwrap_or_else(|| ident.unraw()); let ctx = &Ctx::new(&options.krate); let Ctx { pyo3_path } = ctx; let pyinit_symbol = format!("PyInit_{}", name); diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 179fe71bb9e..06699c80663 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -6,6 +6,8 @@ use crate::attributes::{ ModuleAttribute, NameAttribute, NameLitStr, RenameAllAttribute, }; use crate::deprecations::Deprecations; +#[cfg(feature = "experimental-inspect")] +use crate::introspection::class_introspection_code; use crate::konst::{ConstAttributes, ConstSpec}; use crate::method::{FnArg, FnSpec, PyArg, RegularArg}; use crate::pyfunction::ConstructorAttribute; @@ -903,6 +905,7 @@ fn impl_complex_enum( impl_builder.impl_pyclassimpl(ctx)?, impl_builder.impl_add_to_module(ctx), impl_builder.impl_freelist(ctx), + impl_builder.impl_introspection(ctx), ] .into_iter() .collect(); @@ -1370,17 +1373,17 @@ impl<'a> PyClassImplsBuilder<'a> { } fn impl_all(&self, ctx: &Ctx) -> Result { - let tokens = [ + Ok([ self.impl_pyclass(ctx), self.impl_extractext(ctx), self.impl_into_py(ctx), self.impl_pyclassimpl(ctx)?, self.impl_add_to_module(ctx), self.impl_freelist(ctx), + self.impl_introspection(ctx), ] .into_iter() - .collect(); - Ok(tokens) + .collect()) } fn impl_pyclass(&self, ctx: &Ctx) -> TokenStream { @@ -1682,6 +1685,18 @@ impl<'a> PyClassImplsBuilder<'a> { Vec::new() } } + + #[cfg(feature = "experimental-inspect")] + fn impl_introspection(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; + let name = get_class_python_name(self.cls, self.attr).to_string(); + class_introspection_code(pyo3_path, self.cls, &name) + } + + #[cfg(not(feature = "experimental-inspect"))] + fn impl_introspection(&self, _ctx: &Ctx) -> TokenStream { + quote! {} + } } fn define_inventory_class(inventory_class_name: &syn::Ident, ctx: &Ctx) -> TokenStream { diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index e259f0e2c1e..a78b8a0f43c 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "experimental-inspect")] +use crate::introspection::function_introspection_code; use crate::utils::Ctx; use crate::{ attributes::{ @@ -258,15 +260,19 @@ pub fn impl_wrap_pyfunction( let wrapper_ident = format_ident!("__pyfunction_{}", spec.name); let wrapper = spec.get_wrapper_function(&wrapper_ident, None, ctx)?; let methoddef = spec.get_methoddef(wrapper_ident, &spec.get_doc(&func.attrs), ctx); + #[cfg(feature = "experimental-inspect")] + let introspection = function_introspection_code(pyo3_path, &name.to_string()); + #[cfg(not(feature = "experimental-inspect"))] + let introspection = quote! {}; let wrapped_pyfunction = quote! { - // Create a module with the same name as the `#[pyfunction]` - this way `use ` // will actually bring both the module and the function into scope. #[doc(hidden)] #vis mod #name { pub(crate) struct MakeDef; pub const _PYO3_DEF: #pyo3_path::impl_::pymethods::PyMethodDef = MakeDef::_PYO3_DEF; + #introspection } // Generate the definition inside an anonymous function in the same scope as the original function - diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 690924c76a5..ad2c54143db 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -17,6 +17,7 @@ proc-macro = true multiple-pymethods = [] experimental-async = ["pyo3-macros-backend/experimental-async"] experimental-declarative-modules = [] +experimental-inspect = ["pyo3-macros-backend/experimental-inspect"] [dependencies] proc-macro2 = { version = "1", default-features = false } diff --git a/pytests/Cargo.toml b/pytests/Cargo.toml index 255094a6c40..758764d8c1b 100644 --- a/pytests/Cargo.toml +++ b/pytests/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" publish = false [dependencies] -pyo3 = { path = "../", features = ["extension-module"] } +pyo3 = { path = "../", features = ["extension-module", "experimental-declarative-modules", "experimental-inspect"] } [build-dependencies] pyo3-build-config = { path = "../pyo3-build-config" } diff --git a/pytests/src/lib.rs b/pytests/src/lib.rs index cbd65c8012c..3f3cdd3ed3a 100644 --- a/pytests/src/lib.rs +++ b/pytests/src/lib.rs @@ -18,43 +18,48 @@ pub mod sequence; pub mod subclassing; #[pymodule] -fn pyo3_pytests(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_wrapped(wrap_pymodule!(awaitable::awaitable))?; - #[cfg(not(Py_LIMITED_API))] - m.add_wrapped(wrap_pymodule!(buf_and_str::buf_and_str))?; - m.add_wrapped(wrap_pymodule!(comparisons::comparisons))?; - #[cfg(not(Py_LIMITED_API))] - m.add_wrapped(wrap_pymodule!(datetime::datetime))?; - m.add_wrapped(wrap_pymodule!(dict_iter::dict_iter))?; - m.add_wrapped(wrap_pymodule!(enums::enums))?; - m.add_wrapped(wrap_pymodule!(misc::misc))?; - m.add_wrapped(wrap_pymodule!(objstore::objstore))?; - m.add_wrapped(wrap_pymodule!(othermod::othermod))?; - m.add_wrapped(wrap_pymodule!(path::path))?; - m.add_wrapped(wrap_pymodule!(pyclasses::pyclasses))?; - m.add_wrapped(wrap_pymodule!(pyfunctions::pyfunctions))?; - m.add_wrapped(wrap_pymodule!(sequence::sequence))?; - m.add_wrapped(wrap_pymodule!(subclassing::subclassing))?; +mod pyo3_pytests { + use super::*; + + #[pymodule_export] + use {pyclasses::pyclasses, pyfunctions::pyfunctions}; // Inserting to sys.modules allows importing submodules nicely from Python // e.g. import pyo3_pytests.buf_and_str as bas + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_wrapped(wrap_pymodule!(awaitable::awaitable))?; + #[cfg(not(Py_LIMITED_API))] + m.add_wrapped(wrap_pymodule!(buf_and_str::buf_and_str))?; + m.add_wrapped(wrap_pymodule!(comparisons::comparisons))?; + #[cfg(not(Py_LIMITED_API))] + m.add_wrapped(wrap_pymodule!(datetime::datetime))?; + m.add_wrapped(wrap_pymodule!(dict_iter::dict_iter))?; + m.add_wrapped(wrap_pymodule!(enums::enums))?; + m.add_wrapped(wrap_pymodule!(misc::misc))?; + m.add_wrapped(wrap_pymodule!(objstore::objstore))?; + m.add_wrapped(wrap_pymodule!(othermod::othermod))?; + m.add_wrapped(wrap_pymodule!(path::path))?; + m.add_wrapped(wrap_pymodule!(sequence::sequence))?; + m.add_wrapped(wrap_pymodule!(subclassing::subclassing))?; - let sys = PyModule::import_bound(py, "sys")?; - let sys_modules = sys.getattr("modules")?.downcast_into::()?; - sys_modules.set_item("pyo3_pytests.awaitable", m.getattr("awaitable")?)?; - sys_modules.set_item("pyo3_pytests.buf_and_str", m.getattr("buf_and_str")?)?; - sys_modules.set_item("pyo3_pytests.comparisons", m.getattr("comparisons")?)?; - sys_modules.set_item("pyo3_pytests.datetime", m.getattr("datetime")?)?; - sys_modules.set_item("pyo3_pytests.dict_iter", m.getattr("dict_iter")?)?; - sys_modules.set_item("pyo3_pytests.enums", m.getattr("enums")?)?; - sys_modules.set_item("pyo3_pytests.misc", m.getattr("misc")?)?; - sys_modules.set_item("pyo3_pytests.objstore", m.getattr("objstore")?)?; - sys_modules.set_item("pyo3_pytests.othermod", m.getattr("othermod")?)?; - sys_modules.set_item("pyo3_pytests.path", m.getattr("path")?)?; - sys_modules.set_item("pyo3_pytests.pyclasses", m.getattr("pyclasses")?)?; - sys_modules.set_item("pyo3_pytests.pyfunctions", m.getattr("pyfunctions")?)?; - sys_modules.set_item("pyo3_pytests.sequence", m.getattr("sequence")?)?; - sys_modules.set_item("pyo3_pytests.subclassing", m.getattr("subclassing")?)?; + let sys = PyModule::import_bound(m.py(), "sys")?; + let sys_modules = sys.getattr("modules")?.downcast_into::()?; + sys_modules.set_item("pyo3_pytests.awaitable", m.getattr("awaitable")?)?; + sys_modules.set_item("pyo3_pytests.buf_and_str", m.getattr("buf_and_str")?)?; + sys_modules.set_item("pyo3_pytests.comparisons", m.getattr("comparisons")?)?; + sys_modules.set_item("pyo3_pytests.datetime", m.getattr("datetime")?)?; + sys_modules.set_item("pyo3_pytests.dict_iter", m.getattr("dict_iter")?)?; + sys_modules.set_item("pyo3_pytests.enums", m.getattr("enums")?)?; + sys_modules.set_item("pyo3_pytests.misc", m.getattr("misc")?)?; + sys_modules.set_item("pyo3_pytests.objstore", m.getattr("objstore")?)?; + sys_modules.set_item("pyo3_pytests.othermod", m.getattr("othermod")?)?; + sys_modules.set_item("pyo3_pytests.path", m.getattr("path")?)?; + sys_modules.set_item("pyo3_pytests.pyclasses", m.getattr("pyclasses")?)?; + sys_modules.set_item("pyo3_pytests.pyfunctions", m.getattr("pyfunctions")?)?; + sys_modules.set_item("pyo3_pytests.sequence", m.getattr("sequence")?)?; + sys_modules.set_item("pyo3_pytests.subclassing", m.getattr("subclassing")?)?; - Ok(()) + Ok(()) + } } diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index 6338596b481..8dd9c512542 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -67,11 +67,10 @@ impl AssertingBaseClass { struct ClassWithoutConstructor; #[pymodule] -pub fn pyclasses(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - - Ok(()) +pub mod pyclasses { + #[pymodule_export] + use super::{ + AssertingBaseClass, ClassWithoutConstructor, EmptyClass, + PyClassIter, + }; } diff --git a/src/impl_.rs b/src/impl_.rs index 71ba397cb94..890819a89e1 100644 --- a/src/impl_.rs +++ b/src/impl_.rs @@ -6,6 +6,8 @@ //! APIs may may change at any time without documentation in the CHANGELOG and without //! breaking semver guarantees. +#[cfg(feature = "experimental-inspect")] +pub mod concat; #[cfg(feature = "experimental-async")] pub mod coroutine; pub mod deprecations; diff --git a/src/impl_/concat.rs b/src/impl_/concat.rs new file mode 100644 index 00000000000..0a31300aabc --- /dev/null +++ b/src/impl_/concat.rs @@ -0,0 +1,29 @@ +/// `concat!` but working with constants +#[macro_export] +#[doc(hidden)] +macro_rules! const_concat { + ($e:expr) => {{ + $e + }}; + ($l:expr, $($r:expr),+ $(,)?) => {{ + const L: &'static str = $l; + const R: &'static str = $crate::impl_::concat::const_concat!($($r),*); + const LEN: usize = L.len() + R.len(); + const fn combine(l: &'static [u8], r: &'static [u8]) -> [u8; LEN] { + let mut out = [0u8; LEN]; + let mut i = 0; + while i < l.len() { + out[i] = l[i]; + i += 1; + } + while i < LEN { + out[i] = r[i - l.len()]; + i += 1; + } + out + } + unsafe { ::std::str::from_utf8_unchecked(&combine(L.as_bytes(), R.as_bytes())) } + }} +} + +pub use const_concat; diff --git a/src/types/mod.rs b/src/types/mod.rs index 2203ccdf2dc..d2dba4c4731 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -264,6 +264,10 @@ macro_rules! pyobject_native_type_info( impl $name { #[doc(hidden)] pub const _PYO3_DEF: $crate::impl_::pymodule::AddTypeToModule = $crate::impl_::pymodule::AddTypeToModule::new(); + + #[allow(dead_code)] + #[doc(hidden)] + pub const _PYO3_INTROSPECTION_ID: &'static str = concat!(stringify!($module), stringify!($name)); } }; );