diff --git a/mlir/examples/standalone/python/CMakeLists.txt b/mlir/examples/standalone/python/CMakeLists.txt index 2a4fd99d243e0..905c944939756 100644 --- a/mlir/examples/standalone/python/CMakeLists.txt +++ b/mlir/examples/standalone/python/CMakeLists.txt @@ -74,63 +74,66 @@ add_mlir_python_common_capi_library(StandalonePythonCAPI set(StandalonePythonModules_ROOT_PREFIX "${MLIR_BINARY_DIR}/${MLIR_BINDINGS_PYTHON_INSTALL_PREFIX}") -# Everything here is very tightly coupled. See the ample descriptions at the bottom of -# mlir/python/CMakeLists.txt. - -# For a non-external projects build (e.g., installed distro) the type gen targets for the core _mlir module -# need to be re-declared. On the contrary, for an external projects build, the MLIRPythonExtension.Core.type_stub_gen -# target already exists and can just be added to DECLARED_SOURCES (see below). -if(NOT EXTERNAL_PROJECT_BUILD) - set(_core_type_stub_sources - _mlir/__init__.pyi - _mlir/ir.pyi - _mlir/passmanager.pyi - _mlir/rewrite.pyi - ) - get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES) +if(NOT CMAKE_CROSSCOMPILING) + # Everything here is very tightly coupled. See the ample descriptions at the bottom of + # mlir/python/CMakeLists.txt. + + # For a non-external projects build (e.g., installed distro) the type gen targets for the core _mlir module + # need to be re-declared. On the contrary, for an external projects build, the MLIRPythonExtension.Core.type_stub_gen + # target already exists and can just be added to DECLARED_SOURCES (see below). + if(NOT EXTERNAL_PROJECT_BUILD) + set(_core_type_stub_sources + _mlir/__init__.pyi + _mlir/ir.pyi + _mlir/passmanager.pyi + _mlir/rewrite.pyi + ) + get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES) + mlir_generate_type_stubs( + MODULE_NAME _mlir + DEPENDS_TARGETS StandalonePythonModules.extension._mlir.dso + OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs" + OUTPUTS "${_core_type_stub_sources}" + DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}" + IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/_mlir_libs" + VERBOSE + ) + set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}") + + list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/") + declare_mlir_python_sources( + StandalonePythonExtension.Core.type_stub_gen + ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs" + ADD_TO_PARENT StandalonePythonSources + SOURCES "${_core_type_stub_sources}" + ) + endif() + + get_target_property(_standalone_extension_srcs StandalonePythonSources.NanobindExtension INTERFACE_SOURCES) mlir_generate_type_stubs( - MODULE_NAME _mlir - DEPENDS_TARGETS StandalonePythonModules.extension._mlir.dso + MODULE_NAME mlir_standalone._mlir_libs._standaloneDialectsNanobind + DEPENDS_TARGETS + StandalonePythonModules.extension._mlir.dso + StandalonePythonModules.extension._standaloneDialectsNanobind.dso OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs" - OUTPUTS "${_core_type_stub_sources}" - DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}" - IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/_mlir_libs" - VERBOSE + OUTPUTS + _standaloneDialectsNanobind/__init__.pyi + _standaloneDialectsNanobind/standalone.pyi + DEPENDS_TARGET_SRC_DEPS "${_standalone_extension_srcs}" + IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/.." ) - set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}") + set(_standaloneDialectsNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}") - list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/") declare_mlir_python_sources( - StandalonePythonExtension.Core.type_stub_gen + StandalonePythonSources.type_stub_gen ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs" ADD_TO_PARENT StandalonePythonSources - SOURCES "${_core_type_stub_sources}" + SOURCES + _mlir_libs/_standaloneDialectsNanobind/__init__.pyi + _mlir_libs/_standaloneDialectsNanobind/standalone.pyi ) endif() -get_target_property(_standalone_extension_srcs StandalonePythonSources.NanobindExtension INTERFACE_SOURCES) -mlir_generate_type_stubs( - MODULE_NAME mlir_standalone._mlir_libs._standaloneDialectsNanobind - DEPENDS_TARGETS - StandalonePythonModules.extension._mlir.dso - StandalonePythonModules.extension._standaloneDialectsNanobind.dso - OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs" - OUTPUTS - _standaloneDialectsNanobind/__init__.pyi - _standaloneDialectsNanobind/standalone.pyi - DEPENDS_TARGET_SRC_DEPS "${_standalone_extension_srcs}" - IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/.." -) -set(_standaloneDialectsNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}") - -declare_mlir_python_sources( - StandalonePythonSources.type_stub_gen - ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs" - ADD_TO_PARENT StandalonePythonSources - SOURCES - _mlir_libs/_standaloneDialectsNanobind/__init__.pyi - _mlir_libs/_standaloneDialectsNanobind/standalone.pyi -) set(_declared_sources StandalonePythonSources MLIRPythonSources.Core @@ -138,9 +141,10 @@ set(_declared_sources ) # For an external projects build, the MLIRPythonExtension.Core.type_stub_gen # target already exists and can just be added to DECLARED_SOURCES. -if(EXTERNAL_PROJECT_BUILD) +if(EXTERNAL_PROJECT_BUILD AND (NOT CMAKE_CROSSCOMPILING)) list(APPEND _declared_sources MLIRPythonExtension.Core.type_stub_gen) endif() + add_mlir_python_modules(StandalonePythonModules ROOT_PREFIX "${StandalonePythonModules_ROOT_PREFIX}" INSTALL_PREFIX "${MLIR_BINDINGS_PYTHON_INSTALL_PREFIX}" @@ -148,7 +152,10 @@ add_mlir_python_modules(StandalonePythonModules COMMON_CAPI_LINK_LIBS StandalonePythonCAPI ) -if(NOT EXTERNAL_PROJECT_BUILD) - add_dependencies(StandalonePythonModules "${_mlir_typestub_gen_target}") + +if(NOT CMAKE_CROSSCOMPILING) + if(NOT EXTERNAL_PROJECT_BUILD) + add_dependencies(StandalonePythonModules "${_mlir_typestub_gen_target}") + endif() + add_dependencies(StandalonePythonModules "${_standaloneDialectsNanobind_typestub_gen_target}") endif() -add_dependencies(StandalonePythonModules "${_standaloneDialectsNanobind_typestub_gen_target}") diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt index d6686bb89ce4e..9f5246de6bda0 100644 --- a/mlir/python/CMakeLists.txt +++ b/mlir/python/CMakeLists.txt @@ -873,85 +873,89 @@ if(NOT LLVM_ENABLE_IDE) ) endif() -# _mlir stubgen -# Note: All this needs to come before add_mlir_python_modules(MLIRPythonModules so that the install targets for the -# generated type stubs get created. - -set(_core_type_stub_sources - _mlir/__init__.pyi - _mlir/ir.pyi - _mlir/passmanager.pyi - _mlir/rewrite.pyi -) - -# Note 1: INTERFACE_SOURCES is a genex ($ $) -# which will be evaluated by file(GENERATE ...) inside mlir_generate_type_stubs. This will evaluate to the correct -# thing in the build dir (i.e., actual source dir paths) and in the install dir -# (where it's a conventional path; see install/lib/cmake/mlir/MLIRTargets.cmake). -# -# Note 2: MLIRPythonExtension.Core is the target that is defined using target_sources(INTERFACE) -# **NOT** MLIRPythonModules.extension._mlir.dso. So be sure to use the correct target! -get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES) - -# Why is MODULE_NAME _mlir here but mlir._mlir_libs._mlirPythonTestNanobind below??? -# The _mlir extension can be imported independently of any other python code and/or extension modules. -# I.e., you could do `cd $MLIRPythonModules_ROOT_PREFIX/_mlir_libs && python -c "import _mlir"` (try it!). -# _mlir is also (currently) the only extension for which this is possible because dialect extensions modules, -# which generally make use of `mlir_value_subclass/mlir_type_subclass/mlir_attribute_subclass`, perform an -# `import mlir` right when they're loaded (see the mlir_*_subclass ctors in NanobindAdaptors.h). -# Note, this also why IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs" here while below -# "${MLIRPythonModules_ROOT_PREFIX}/.." (because MLIR_BINDINGS_PYTHON_INSTALL_PREFIX, by default, ends at mlir). -# -# Further note: this function creates file targets like -# "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs/_mlir/__init__.pyi". These must match the file targets -# that declare_mlir_python_sources expects, which are like "${ROOT_DIR}/${WHATEVER_SOURCE}". -# This is why _mlir_libs is prepended below. -mlir_generate_type_stubs( - MODULE_NAME _mlir - DEPENDS_TARGETS MLIRPythonModules.extension._mlir.dso - OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs" - OUTPUTS "${_core_type_stub_sources}" - DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}" - IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs" -) -set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}") - -list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/") -# Note, we do not do ADD_TO_PARENT here so that the type stubs are not associated (as mlir_DEPENDS) with -# MLIRPythonSources.Core (or something) when a distro is installed/created. Otherwise they would not be regenerated -# by users of the distro (the stubs are still installed in the distro - they are just not added to mlir_DEPENDS). -declare_mlir_python_sources( - MLIRPythonExtension.Core.type_stub_gen - ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs" - SOURCES "${_core_type_stub_sources}" -) - -# _mlirPythonTestNanobind stubgen +# Stubgen doesn't work when cross-compiling (stubgen will run in the host interpreter and then fail +# to find the extension module for the host arch). +if(NOT CMAKE_CROSSCOMPILING) + # _mlir stubgen + # Note: All this needs to come before add_mlir_python_modules(MLIRPythonModules so that the install targets for the + # generated type stubs get created. + + set(_core_type_stub_sources + _mlir/__init__.pyi + _mlir/ir.pyi + _mlir/passmanager.pyi + _mlir/rewrite.pyi + ) -if(MLIR_INCLUDE_TESTS) - get_target_property(_test_extension_srcs MLIRPythonTestSources.PythonTestExtensionNanobind INTERFACE_SOURCES) + # Note 1: INTERFACE_SOURCES is a genex ($ $) + # which will be evaluated by file(GENERATE ...) inside mlir_generate_type_stubs. This will evaluate to the correct + # thing in the build dir (i.e., actual source dir paths) and in the install dir + # (where it's a conventional path; see install/lib/cmake/mlir/MLIRTargets.cmake). + # + # Note 2: MLIRPythonExtension.Core is the target that is defined using target_sources(INTERFACE) + # **NOT** MLIRPythonModules.extension._mlir.dso. So be sure to use the correct target! + get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES) + + # Why is MODULE_NAME _mlir here but mlir._mlir_libs._mlirPythonTestNanobind below??? + # The _mlir extension can be imported independently of any other python code and/or extension modules. + # I.e., you could do `cd $MLIRPythonModules_ROOT_PREFIX/_mlir_libs && python -c "import _mlir"` (try it!). + # _mlir is also (currently) the only extension for which this is possible because dialect extensions modules, + # which generally make use of `mlir_value_subclass/mlir_type_subclass/mlir_attribute_subclass`, perform an + # `import mlir` right when they're loaded (see the mlir_*_subclass ctors in NanobindAdaptors.h). + # Note, this also why IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs" here while below + # "${MLIRPythonModules_ROOT_PREFIX}/.." (because MLIR_BINDINGS_PYTHON_INSTALL_PREFIX, by default, ends at mlir). + # + # Further note: this function creates file targets like + # "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs/_mlir/__init__.pyi". These must match the file targets + # that declare_mlir_python_sources expects, which are like "${ROOT_DIR}/${WHATEVER_SOURCE}". + # This is why _mlir_libs is prepended below. mlir_generate_type_stubs( - # This is the FQN path because dialect modules import _mlir when loaded. See above. - MODULE_NAME mlir._mlir_libs._mlirPythonTestNanobind - DEPENDS_TARGETS - # You need both _mlir and _mlirPythonTestNanobind because dialect modules import _mlir when loaded - # (so _mlir needs to be built before calling stubgen). - MLIRPythonModules.extension._mlir.dso - MLIRPythonModules.extension._mlirPythonTestNanobind.dso - # You need this one so that ir.py "built" because mlir._mlir_libs.__init__.py import mlir.ir in _site_initialize. - MLIRPythonModules.sources.MLIRPythonSources.Core.Python + MODULE_NAME _mlir + DEPENDS_TARGETS MLIRPythonModules.extension._mlir.dso OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs" - OUTPUTS _mlirPythonTestNanobind.pyi - DEPENDS_TARGET_SRC_DEPS "${_test_extension_srcs}" - IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/.." + OUTPUTS "${_core_type_stub_sources}" + DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}" + IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs" ) - set(_mlirPythonTestNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}") + set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}") + + list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/") + # Note, we do not do ADD_TO_PARENT here so that the type stubs are not associated (as mlir_DEPENDS) with + # MLIRPythonSources.Core (or something) when a distro is installed/created. Otherwise they would not be regenerated + # by users of the distro (the stubs are still installed in the distro - they are just not added to mlir_DEPENDS). declare_mlir_python_sources( - MLIRPythonTestSources.PythonTestExtensionNanobind.type_stub_gen + MLIRPythonExtension.Core.type_stub_gen ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs" - ADD_TO_PARENT MLIRPythonTestSources.Dialects - SOURCES _mlir_libs/_mlirPythonTestNanobind.pyi + SOURCES "${_core_type_stub_sources}" ) + + # _mlirPythonTestNanobind stubgen + + if(MLIR_INCLUDE_TESTS) + get_target_property(_test_extension_srcs MLIRPythonTestSources.PythonTestExtensionNanobind INTERFACE_SOURCES) + mlir_generate_type_stubs( + # This is the FQN path because dialect modules import _mlir when loaded. See above. + MODULE_NAME mlir._mlir_libs._mlirPythonTestNanobind + DEPENDS_TARGETS + # You need both _mlir and _mlirPythonTestNanobind because dialect modules import _mlir when loaded + # (so _mlir needs to be built before calling stubgen). + MLIRPythonModules.extension._mlir.dso + MLIRPythonModules.extension._mlirPythonTestNanobind.dso + # You need this one so that ir.py "built" because mlir._mlir_libs.__init__.py import mlir.ir in _site_initialize. + MLIRPythonModules.sources.MLIRPythonSources.Core.Python + OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs" + OUTPUTS _mlirPythonTestNanobind.pyi + DEPENDS_TARGET_SRC_DEPS "${_test_extension_srcs}" + IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/.." + ) + set(_mlirPythonTestNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}") + declare_mlir_python_sources( + MLIRPythonTestSources.PythonTestExtensionNanobind.type_stub_gen + ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs" + ADD_TO_PARENT MLIRPythonTestSources.Dialects + SOURCES _mlir_libs/_mlirPythonTestNanobind.pyi + ) + endif() endif() ################################################################################ @@ -959,18 +963,23 @@ endif() # This must come last. ################################################################################ +set(_declared_sources MLIRPythonSources MLIRPythonExtension.RegisterEverything) +if(NOT CMAKE_CROSSCOMPILING) + list(APPEND _declared_sources MLIRPythonExtension.Core.type_stub_gen) +endif() + add_mlir_python_modules(MLIRPythonModules ROOT_PREFIX ${MLIRPythonModules_ROOT_PREFIX} INSTALL_PREFIX "${MLIR_BINDINGS_PYTHON_INSTALL_PREFIX}" DECLARED_SOURCES - MLIRPythonSources - MLIRPythonExtension.RegisterEverything - MLIRPythonExtension.Core.type_stub_gen + ${_declared_sources} ${_ADDL_TEST_SOURCES} COMMON_CAPI_LINK_LIBS MLIRPythonCAPI ) -add_dependencies(MLIRPythonModules "${_mlir_typestub_gen_target}") -if(MLIR_INCLUDE_TESTS) - add_dependencies(MLIRPythonModules "${_mlirPythonTestNanobind_typestub_gen_target}") +if(NOT CMAKE_CROSSCOMPILING) + add_dependencies(MLIRPythonModules "${_mlir_typestub_gen_target}") + if(MLIR_INCLUDE_TESTS) + add_dependencies(MLIRPythonModules "${_mlirPythonTestNanobind_typestub_gen_target}") + endif() endif()