[MLIR][Python] don't generate type stubs when cross-compiling #160793
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
makslevental
requested review from
ftynse,
stellaraccident and
rolfmorel
as code owners
|
In reality you can't really build the Python bindings when cross-compiling anyway (because they depend on plat specific |
|
@llvm/pr-subscribers-mlir Author: Maksim Levental (makslevental) ChangesStubgen doesn't work when cross-compiling (stubgen will run in the host interpreter and then fail to find the extension module for the host arch). So disable it. Full diff: https://github.com/llvm/llvm-project/pull/160793.diff 2 Files Affected:
diff --git a/mlir/examples/standalone/python/CMakeLists.txt b/mlir/examples/standalone/python/CMakeLists.txt
index 2a4fd99d243e0..905c944939756 100644
--- a/mlir/examples/standalone/python/CMakeLists.txt
+++ b/mlir/examples/standalone/python/CMakeLists.txt
@@ -74,63 +74,66 @@ add_mlir_python_common_capi_library(StandalonePythonCAPI
set(StandalonePythonModules_ROOT_PREFIX "${MLIR_BINARY_DIR}/${MLIR_BINDINGS_PYTHON_INSTALL_PREFIX}")
-# Everything here is very tightly coupled. See the ample descriptions at the bottom of
-# mlir/python/CMakeLists.txt.
-
-# For a non-external projects build (e.g., installed distro) the type gen targets for the core _mlir module
-# need to be re-declared. On the contrary, for an external projects build, the MLIRPythonExtension.Core.type_stub_gen
-# target already exists and can just be added to DECLARED_SOURCES (see below).
-if(NOT EXTERNAL_PROJECT_BUILD)
- set(_core_type_stub_sources
- _mlir/__init__.pyi
- _mlir/ir.pyi
- _mlir/passmanager.pyi
- _mlir/rewrite.pyi
- )
- get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES)
+if(NOT CMAKE_CROSSCOMPILING)
+ # Everything here is very tightly coupled. See the ample descriptions at the bottom of
+ # mlir/python/CMakeLists.txt.
+
+ # For a non-external projects build (e.g., installed distro) the type gen targets for the core _mlir module
+ # need to be re-declared. On the contrary, for an external projects build, the MLIRPythonExtension.Core.type_stub_gen
+ # target already exists and can just be added to DECLARED_SOURCES (see below).
+ if(NOT EXTERNAL_PROJECT_BUILD)
+ set(_core_type_stub_sources
+ _mlir/__init__.pyi
+ _mlir/ir.pyi
+ _mlir/passmanager.pyi
+ _mlir/rewrite.pyi
+ )
+ get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES)
+ mlir_generate_type_stubs(
+ MODULE_NAME _mlir
+ DEPENDS_TARGETS StandalonePythonModules.extension._mlir.dso
+ OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
+ OUTPUTS "${_core_type_stub_sources}"
+ DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}"
+ IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/_mlir_libs"
+ VERBOSE
+ )
+ set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
+
+ list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/")
+ declare_mlir_python_sources(
+ StandalonePythonExtension.Core.type_stub_gen
+ ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
+ ADD_TO_PARENT StandalonePythonSources
+ SOURCES "${_core_type_stub_sources}"
+ )
+ endif()
+
+ get_target_property(_standalone_extension_srcs StandalonePythonSources.NanobindExtension INTERFACE_SOURCES)
mlir_generate_type_stubs(
- MODULE_NAME _mlir
- DEPENDS_TARGETS StandalonePythonModules.extension._mlir.dso
+ MODULE_NAME mlir_standalone._mlir_libs._standaloneDialectsNanobind
+ DEPENDS_TARGETS
+ StandalonePythonModules.extension._mlir.dso
+ StandalonePythonModules.extension._standaloneDialectsNanobind.dso
OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
- OUTPUTS "${_core_type_stub_sources}"
- DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}"
- IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/_mlir_libs"
- VERBOSE
+ OUTPUTS
+ _standaloneDialectsNanobind/__init__.pyi
+ _standaloneDialectsNanobind/standalone.pyi
+ DEPENDS_TARGET_SRC_DEPS "${_standalone_extension_srcs}"
+ IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/.."
)
- set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
+ set(_standaloneDialectsNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
- list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/")
declare_mlir_python_sources(
- StandalonePythonExtension.Core.type_stub_gen
+ StandalonePythonSources.type_stub_gen
ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
ADD_TO_PARENT StandalonePythonSources
- SOURCES "${_core_type_stub_sources}"
+ SOURCES
+ _mlir_libs/_standaloneDialectsNanobind/__init__.pyi
+ _mlir_libs/_standaloneDialectsNanobind/standalone.pyi
)
endif()
-get_target_property(_standalone_extension_srcs StandalonePythonSources.NanobindExtension INTERFACE_SOURCES)
-mlir_generate_type_stubs(
- MODULE_NAME mlir_standalone._mlir_libs._standaloneDialectsNanobind
- DEPENDS_TARGETS
- StandalonePythonModules.extension._mlir.dso
- StandalonePythonModules.extension._standaloneDialectsNanobind.dso
- OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
- OUTPUTS
- _standaloneDialectsNanobind/__init__.pyi
- _standaloneDialectsNanobind/standalone.pyi
- DEPENDS_TARGET_SRC_DEPS "${_standalone_extension_srcs}"
- IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/.."
-)
-set(_standaloneDialectsNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
-
-declare_mlir_python_sources(
- StandalonePythonSources.type_stub_gen
- ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
- ADD_TO_PARENT StandalonePythonSources
- SOURCES
- _mlir_libs/_standaloneDialectsNanobind/__init__.pyi
- _mlir_libs/_standaloneDialectsNanobind/standalone.pyi
-)
set(_declared_sources
StandalonePythonSources
MLIRPythonSources.Core
@@ -138,9 +141,10 @@ set(_declared_sources
)
# For an external projects build, the MLIRPythonExtension.Core.type_stub_gen
# target already exists and can just be added to DECLARED_SOURCES.
-if(EXTERNAL_PROJECT_BUILD)
+if(EXTERNAL_PROJECT_BUILD AND (NOT CMAKE_CROSSCOMPILING))
list(APPEND _declared_sources MLIRPythonExtension.Core.type_stub_gen)
endif()
+
add_mlir_python_modules(StandalonePythonModules
ROOT_PREFIX "${StandalonePythonModules_ROOT_PREFIX}"
INSTALL_PREFIX "${MLIR_BINDINGS_PYTHON_INSTALL_PREFIX}"
@@ -148,7 +152,10 @@ add_mlir_python_modules(StandalonePythonModules
COMMON_CAPI_LINK_LIBS
StandalonePythonCAPI
)
-if(NOT EXTERNAL_PROJECT_BUILD)
- add_dependencies(StandalonePythonModules "${_mlir_typestub_gen_target}")
+
+if(NOT CMAKE_CROSSCOMPILING)
+ if(NOT EXTERNAL_PROJECT_BUILD)
+ add_dependencies(StandalonePythonModules "${_mlir_typestub_gen_target}")
+ endif()
+ add_dependencies(StandalonePythonModules "${_standaloneDialectsNanobind_typestub_gen_target}")
endif()
-add_dependencies(StandalonePythonModules "${_standaloneDialectsNanobind_typestub_gen_target}")
diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt
index d6686bb89ce4e..16d1eaf19aa8c 100644
--- a/mlir/python/CMakeLists.txt
+++ b/mlir/python/CMakeLists.txt
@@ -873,85 +873,87 @@ if(NOT LLVM_ENABLE_IDE)
)
endif()
-# _mlir stubgen
-# Note: All this needs to come before add_mlir_python_modules(MLIRPythonModules so that the install targets for the
-# generated type stubs get created.
-
-set(_core_type_stub_sources
- _mlir/__init__.pyi
- _mlir/ir.pyi
- _mlir/passmanager.pyi
- _mlir/rewrite.pyi
-)
-
-# Note 1: INTERFACE_SOURCES is a genex ($<BUILD_INTERFACE> $<INSTALL_INTERFACE>)
-# which will be evaluated by file(GENERATE ...) inside mlir_generate_type_stubs. This will evaluate to the correct
-# thing in the build dir (i.e., actual source dir paths) and in the install dir
-# (where it's a conventional path; see install/lib/cmake/mlir/MLIRTargets.cmake).
-#
-# Note 2: MLIRPythonExtension.Core is the target that is defined using target_sources(INTERFACE)
-# **NOT** MLIRPythonModules.extension._mlir.dso. So be sure to use the correct target!
-get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES)
-
-# Why is MODULE_NAME _mlir here but mlir._mlir_libs._mlirPythonTestNanobind below???
-# The _mlir extension can be imported independently of any other python code and/or extension modules.
-# I.e., you could do `cd $MLIRPythonModules_ROOT_PREFIX/_mlir_libs && python -c "import _mlir"` (try it!).
-# _mlir is also (currently) the only extension for which this is possible because dialect extensions modules,
-# which generally make use of `mlir_value_subclass/mlir_type_subclass/mlir_attribute_subclass`, perform an
-# `import mlir` right when they're loaded (see the mlir_*_subclass ctors in NanobindAdaptors.h).
-# Note, this also why IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs" here while below
-# "${MLIRPythonModules_ROOT_PREFIX}/.." (because MLIR_BINDINGS_PYTHON_INSTALL_PREFIX, by default, ends at mlir).
-#
-# Further note: this function creates file targets like
-# "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs/_mlir/__init__.pyi". These must match the file targets
-# that declare_mlir_python_sources expects, which are like "${ROOT_DIR}/${WHATEVER_SOURCE}".
-# This is why _mlir_libs is prepended below.
-mlir_generate_type_stubs(
- MODULE_NAME _mlir
- DEPENDS_TARGETS MLIRPythonModules.extension._mlir.dso
- OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
- OUTPUTS "${_core_type_stub_sources}"
- DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}"
- IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs"
-)
-set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
-
-list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/")
-# Note, we do not do ADD_TO_PARENT here so that the type stubs are not associated (as mlir_DEPENDS) with
-# MLIRPythonSources.Core (or something) when a distro is installed/created. Otherwise they would not be regenerated
-# by users of the distro (the stubs are still installed in the distro - they are just not added to mlir_DEPENDS).
-declare_mlir_python_sources(
- MLIRPythonExtension.Core.type_stub_gen
- ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
- SOURCES "${_core_type_stub_sources}"
-)
-
-# _mlirPythonTestNanobind stubgen
+if(NOT CMAKE_CROSSCOMPILING)
+ # _mlir stubgen
+ # Note: All this needs to come before add_mlir_python_modules(MLIRPythonModules so that the install targets for the
+ # generated type stubs get created.
+
+ set(_core_type_stub_sources
+ _mlir/__init__.pyi
+ _mlir/ir.pyi
+ _mlir/passmanager.pyi
+ _mlir/rewrite.pyi
+ )
-if(MLIR_INCLUDE_TESTS)
- get_target_property(_test_extension_srcs MLIRPythonTestSources.PythonTestExtensionNanobind INTERFACE_SOURCES)
+ # Note 1: INTERFACE_SOURCES is a genex ($<BUILD_INTERFACE> $<INSTALL_INTERFACE>)
+ # which will be evaluated by file(GENERATE ...) inside mlir_generate_type_stubs. This will evaluate to the correct
+ # thing in the build dir (i.e., actual source dir paths) and in the install dir
+ # (where it's a conventional path; see install/lib/cmake/mlir/MLIRTargets.cmake).
+ #
+ # Note 2: MLIRPythonExtension.Core is the target that is defined using target_sources(INTERFACE)
+ # **NOT** MLIRPythonModules.extension._mlir.dso. So be sure to use the correct target!
+ get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES)
+
+ # Why is MODULE_NAME _mlir here but mlir._mlir_libs._mlirPythonTestNanobind below???
+ # The _mlir extension can be imported independently of any other python code and/or extension modules.
+ # I.e., you could do `cd $MLIRPythonModules_ROOT_PREFIX/_mlir_libs && python -c "import _mlir"` (try it!).
+ # _mlir is also (currently) the only extension for which this is possible because dialect extensions modules,
+ # which generally make use of `mlir_value_subclass/mlir_type_subclass/mlir_attribute_subclass`, perform an
+ # `import mlir` right when they're loaded (see the mlir_*_subclass ctors in NanobindAdaptors.h).
+ # Note, this also why IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs" here while below
+ # "${MLIRPythonModules_ROOT_PREFIX}/.." (because MLIR_BINDINGS_PYTHON_INSTALL_PREFIX, by default, ends at mlir).
+ #
+ # Further note: this function creates file targets like
+ # "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs/_mlir/__init__.pyi". These must match the file targets
+ # that declare_mlir_python_sources expects, which are like "${ROOT_DIR}/${WHATEVER_SOURCE}".
+ # This is why _mlir_libs is prepended below.
mlir_generate_type_stubs(
- # This is the FQN path because dialect modules import _mlir when loaded. See above.
- MODULE_NAME mlir._mlir_libs._mlirPythonTestNanobind
- DEPENDS_TARGETS
- # You need both _mlir and _mlirPythonTestNanobind because dialect modules import _mlir when loaded
- # (so _mlir needs to be built before calling stubgen).
- MLIRPythonModules.extension._mlir.dso
- MLIRPythonModules.extension._mlirPythonTestNanobind.dso
- # You need this one so that ir.py "built" because mlir._mlir_libs.__init__.py import mlir.ir in _site_initialize.
- MLIRPythonModules.sources.MLIRPythonSources.Core.Python
+ MODULE_NAME _mlir
+ DEPENDS_TARGETS MLIRPythonModules.extension._mlir.dso
OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
- OUTPUTS _mlirPythonTestNanobind.pyi
- DEPENDS_TARGET_SRC_DEPS "${_test_extension_srcs}"
- IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/.."
+ OUTPUTS "${_core_type_stub_sources}"
+ DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}"
+ IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs"
)
- set(_mlirPythonTestNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
+ set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
+
+ list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/")
+ # Note, we do not do ADD_TO_PARENT here so that the type stubs are not associated (as mlir_DEPENDS) with
+ # MLIRPythonSources.Core (or something) when a distro is installed/created. Otherwise they would not be regenerated
+ # by users of the distro (the stubs are still installed in the distro - they are just not added to mlir_DEPENDS).
declare_mlir_python_sources(
- MLIRPythonTestSources.PythonTestExtensionNanobind.type_stub_gen
+ MLIRPythonExtension.Core.type_stub_gen
ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
- ADD_TO_PARENT MLIRPythonTestSources.Dialects
- SOURCES _mlir_libs/_mlirPythonTestNanobind.pyi
+ SOURCES "${_core_type_stub_sources}"
)
+
+ # _mlirPythonTestNanobind stubgen
+
+ if(MLIR_INCLUDE_TESTS)
+ get_target_property(_test_extension_srcs MLIRPythonTestSources.PythonTestExtensionNanobind INTERFACE_SOURCES)
+ mlir_generate_type_stubs(
+ # This is the FQN path because dialect modules import _mlir when loaded. See above.
+ MODULE_NAME mlir._mlir_libs._mlirPythonTestNanobind
+ DEPENDS_TARGETS
+ # You need both _mlir and _mlirPythonTestNanobind because dialect modules import _mlir when loaded
+ # (so _mlir needs to be built before calling stubgen).
+ MLIRPythonModules.extension._mlir.dso
+ MLIRPythonModules.extension._mlirPythonTestNanobind.dso
+ # You need this one so that ir.py "built" because mlir._mlir_libs.__init__.py import mlir.ir in _site_initialize.
+ MLIRPythonModules.sources.MLIRPythonSources.Core.Python
+ OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
+ OUTPUTS _mlirPythonTestNanobind.pyi
+ DEPENDS_TARGET_SRC_DEPS "${_test_extension_srcs}"
+ IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/.."
+ )
+ set(_mlirPythonTestNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
+ declare_mlir_python_sources(
+ MLIRPythonTestSources.PythonTestExtensionNanobind.type_stub_gen
+ ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
+ ADD_TO_PARENT MLIRPythonTestSources.Dialects
+ SOURCES _mlir_libs/_mlirPythonTestNanobind.pyi
+ )
+ endif()
endif()
################################################################################
@@ -959,18 +961,23 @@ endif()
# This must come last.
################################################################################
+set(_declared_sources MLIRPythonSources MLIRPythonExtension.RegisterEverything)
+if(NOT CMAKE_CROSSCOMPILING)
+ list(APPEND _declared_sources MLIRPythonExtension.Core.type_stub_gen)
+endif()
+
add_mlir_python_modules(MLIRPythonModules
ROOT_PREFIX ${MLIRPythonModules_ROOT_PREFIX}
INSTALL_PREFIX "${MLIR_BINDINGS_PYTHON_INSTALL_PREFIX}"
DECLARED_SOURCES
- MLIRPythonSources
- MLIRPythonExtension.RegisterEverything
- MLIRPythonExtension.Core.type_stub_gen
+ ${_declared_sources}
${_ADDL_TEST_SOURCES}
COMMON_CAPI_LINK_LIBS
MLIRPythonCAPI
)
-add_dependencies(MLIRPythonModules "${_mlir_typestub_gen_target}")
-if(MLIR_INCLUDE_TESTS)
- add_dependencies(MLIRPythonModules "${_mlirPythonTestNanobind_typestub_gen_target}")
+if(NOT CMAKE_CROSSCOMPILING)
+ add_dependencies(MLIRPythonModules "${_mlir_typestub_gen_target}")
+ if(MLIR_INCLUDE_TESTS)
+ add_dependencies(MLIRPythonModules "${_mlirPythonTestNanobind_typestub_gen_target}")
+ endif()
endif()
|
makslevental
force-pushed
the
users/makslevental/fix-stubgen-cross-compile
branch
from
3161779 to
9c2f31d
Compare
makslevental
deleted the
users/makslevental/fix-stubgen-cross-compile
branch
|
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/116/builds/18888 Here is the relevant piece of the build log for the reference |
mahesh-attarde
pushed a commit
to mahesh-attarde/llvm-project
that referenced
this pull request
Oct 3, 2025
…60793) Stubgen doesn't work when cross-compiling (stubgen will run in the host interpreter and then fail to find the extension module for the host arch). So disable it when `CMAKE_CROSSCOMPILING=ON`.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Stubgen doesn't work when cross-compiling (stubgen will run in the host interpreter and then fail to find the extension module for the host arch). So disable it when
CMAKE_CROSSCOMPILING=ON.