[MLIR][Python] add Python wheel build demo/test #160388
Merged
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
makslevental
force-pushed
the
users/makslevental/standalone-whee
branch
from
39172ac to
a9d0caf
Compare
|
✅ With the latest revision this PR passed the Python code formatter. |
makslevental
force-pushed
the
users/makslevental/standalone-whee
branch
11 times, most recently
from
118298b to
9e6c475
Compare
makslevental
force-pushed
the
users/makslevental/standalone-whee
branch
16 times, most recently
from
2eec903 to
c385012
Compare
makslevental
force-pushed
the
users/makslevental/standalone-whee
branch
5 times, most recently
from
026c364 to
5ea0008
Compare
|
@llvm/pr-subscribers-mlir Author: Maksim Levental (makslevental) ChangesThis PR demos and tests building Python wheels using scikit-build-core. The test is added to standalone and thus demos "out-of-tree" use cases but the same Full diff: https://github.com/llvm/llvm-project/pull/160388.diff 5 Files Affected:
diff --git a/mlir/examples/standalone/CMakeLists.txt b/mlir/examples/standalone/CMakeLists.txt
index e2bcda7fa6f0b..c6c49fde12d2e 100644
--- a/mlir/examples/standalone/CMakeLists.txt
+++ b/mlir/examples/standalone/CMakeLists.txt
@@ -63,8 +63,12 @@ if(MLIR_ENABLE_BINDINGS_PYTHON)
include(MLIRDetectPythonEnv)
mlir_configure_python_dev_packages()
# Note: for EXTERNAL_PROJECT_BUILD this must be set from the command line.
- set(MLIR_PYTHON_PACKAGE_PREFIX "mlir_standalone" CACHE STRING "" FORCE)
- set(MLIR_BINDINGS_PYTHON_INSTALL_PREFIX "python_packages/standalone/${MLIR_PYTHON_PACKAGE_PREFIX}" CACHE STRING "" FORCE)
+ if(NOT MLIR_PYTHON_PACKAGE_PREFIX)
+ set(MLIR_PYTHON_PACKAGE_PREFIX "mlir_standalone" CACHE STRING "" FORCE)
+ endif()
+ if(NOT MLIR_BINDINGS_PYTHON_INSTALL_PREFIX)
+ set(MLIR_BINDINGS_PYTHON_INSTALL_PREFIX "python_packages/standalone/${MLIR_PYTHON_PACKAGE_PREFIX}" CACHE STRING "" FORCE)
+ endif()
add_subdirectory(python)
endif()
add_subdirectory(test)
diff --git a/mlir/examples/standalone/pyproject.toml b/mlir/examples/standalone/pyproject.toml
new file mode 100644
index 0000000000000..1b3d42bcd8510
--- /dev/null
+++ b/mlir/examples/standalone/pyproject.toml
@@ -0,0 +1,64 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+# Copyright (c) 2025.
+
+[project]
+name = "standalone-python-bindings"
+dynamic = ["version"]
+requires-python = ">=3.8,<=3.14"
+dependencies = [
+ "numpy>=1.19.5, <=2.1.2",
+ "PyYAML>=5.4.0, <=6.0.1",
+ "ml_dtypes>=0.1.0, <=0.6.0; python_version<'3.13'",
+ "ml_dtypes>=0.5.0, <=0.6.0; python_version>='3.13'",
+]
+
+[project.urls]
+Homepage = "https://github.com/llvm/llvm-project"
+Discussions = "https://discourse.llvm.org/"
+"Issue Tracker" = "https://github.com/llvm/llvm-project/issues?q=is%3Aissue%20state%3Aopen%20label%3Amlir%3Apython%20"
+"Source Code" = "https://github.com/llvm/llvm-project/tree/main/mlir/python"
+
+[build-system]
+requires = [
+ "scikit-build-core>=0.10.7",
+ "typing_extensions>=4.12.2",
+ "nanobind>=2.9, <3.0",
+ "pybind11>=2.10.0, <=2.13.6",
+]
+build-backend = "scikit_build_core.build"
+
+[tool.scikit-build]
+# This is the minimum version of scikit-build-core.
+minimum-version = "0.10"
+# This pyproject.toml must be adjacent to the root CMakeLists.txt (wherever project(...) is specified).
+cmake.source-dir = "."
+# This is for installing/distributing the python bindings target and only the python bindings target.
+build.targets = ["StandalonePythonModules"]
+install.components = ["StandalonePythonModules"]
+
+[tool.scikit-build.cmake.define]
+# Optional
+CMAKE_C_COMPILER = { env = "CMAKE_C_COMPILER", default = "" }
+CMAKE_CXX_COMPILER = { env = "CMAKE_CXX_COMPILER", default = "" }
+CMAKE_C_COMPILER_LAUNCHER = { env = "CMAKE_C_COMPILER_LAUNCHER", default = "" }
+CMAKE_CXX_COMPILER_LAUNCHER = { env = "CMAKE_CXX_COMPILER_LAUNCHER", default = "" }
+CMAKE_GENERATOR = { env = "CMAKE_GENERATOR", default = "Ninja" }
+LLVM_USE_LINKER = { env = "LLVM_USE_LINKER", default = "" }
+# Optional but highly recommended.
+CMAKE_VISIBILITY_INLINES_HIDDEN = "ON"
+CMAKE_C_VISIBILITY_PRESET = "hidden"
+CMAKE_CXX_VISIBILITY_PRESET = "hidden"
+
+# Non-optinal (alternatively you could use CMAKE_PREFIX_PATH here).
+MLIR_DIR = { env = "MLIR_DIR", default = "" }
+# Non-optinal
+CMAKE_BUILD_TYPE = { env = "CMAKE_BUILD_TYPE", default = "Release" }
+MLIR_ENABLE_BINDINGS_PYTHON = "ON"
+# Effectively non-optional (any downstream project should specify this).
+MLIR_PYTHON_PACKAGE_PREFIX = "mlir_standalone"
+# This specifies the directory in the install directory (i.e., /tmp/pip-wheel/platlib) where _mlir_libs, dialects, etc.
+# are installed. Thus, this will be the package location (and the name of the package) that pip assumes is
+# the root package.
+MLIR_BINDINGS_PYTHON_INSTALL_PREFIX = "mlir_standalone"
diff --git a/mlir/test/Examples/standalone/lit.local.cfg b/mlir/test/Examples/standalone/lit.local.cfg
index 3b12dcbd99e83..e739ec1ba1052 100644
--- a/mlir/test/Examples/standalone/lit.local.cfg
+++ b/mlir/test/Examples/standalone/lit.local.cfg
@@ -1,3 +1,5 @@
+import os
+
# Disable with sanitizers for now, this require some more setup apparently.
for san in ["asan", "msan", "ubsan"]:
if san in config.available_features:
@@ -7,7 +9,13 @@ config.substitutions.append(("%cmake_exe", config.host_cmake))
config.substitutions.append(("%cmake_generator", config.host_cmake_generator))
config.substitutions.append(("%host_cxx", config.host_cxx))
config.substitutions.append(("%host_cc", config.host_cc))
+config.substitutions.append(("%hostc_compiler_launcher", config.host_c_compiler_launcher))
+config.substitutions.append(("%hostcxx_compiler_launcher", config.host_cxx_compiler_launcher))
config.substitutions.append(("%enable_libcxx", config.enable_libcxx))
config.substitutions.append(("%mlir_cmake_dir", config.mlir_cmake_dir))
+config.substitutions.append(("%mlir_obj_root", config.mlir_obj_root))
config.substitutions.append(("%llvm_use_linker", config.llvm_use_linker))
config.substitutions.append(("%cmake_build_type", config.cmake_build_type))
+
+if "PIP_BREAK_SYSTEM_PACKAGES" in os.environ:
+ config.environment["PIP_BREAK_SYSTEM_PACKAGES"] = os.environ["PIP_BREAK_SYSTEM_PACKAGES"]
diff --git a/mlir/test/Examples/standalone/test.wheel.toy b/mlir/test/Examples/standalone/test.wheel.toy
new file mode 100644
index 0000000000000..61d11914655e7
--- /dev/null
+++ b/mlir/test/Examples/standalone/test.wheel.toy
@@ -0,0 +1,28 @@
+# There's no real issue with windows here, it's just that some CMake generated paths for targets end up being longer
+# than 255 chars when combined with the fact that pip wants to install into a tmp directory buried under
+# C/Users/ContainerAdministrator/AppData/Local/Temp.
+# REQUIRES: system-linux
+
+# RUN: export CMAKE_BUILD_TYPE=%cmake_build_type
+# RUN: export CMAKE_CXX_COMPILER=%host_cxx
+# RUN: export CMAKE_CXX_COMPILER_LAUNCHER=%hostcxx_compiler_launcher
+# RUN: export CMAKE_C_COMPILER=%host_cc
+# RUN: export CMAKE_C_COMPILER_LAUNCHER=%hostc_compiler_launcher
+# RUN: export CMAKE_GENERATOR=%cmake_generator
+# RUN: export LLVM_USE_LINKER=%llvm_use_linker
+# RUN: export MLIR_DIR="%mlir_cmake_dir"
+
+# RUN: %python -m pip uninstall -y standalone-python-bindings
+# RUN: %python -m pip wheel "%mlir_src_root/examples/standalone" -w "%mlir_obj_root/wheelhouse" -v | tee %t
+# RUN: %python -m pip install standalone_python_bindings -f "%mlir_obj_root/wheelhouse" -v | tee -a %t
+# RUN: %python "%mlir_src_root/examples/standalone/test/python/smoketest.py" nanobind | tee -a %t
+
+# RUN: FileCheck --input-file=%t %s
+
+# CHECK: Successfully built standalone-python-bindings
+
+# CHECK: module {
+# CHECK: %[[C2:.*]] = arith.constant 2 : i32
+# CHECK: %[[V0:.*]] = standalone.foo %[[C2]] : i32
+# CHECK: }
+
diff --git a/mlir/test/lit.site.cfg.py.in b/mlir/test/lit.site.cfg.py.in
index 2fc595dfabbf5..940e2ad3c4365 100644
--- a/mlir/test/lit.site.cfg.py.in
+++ b/mlir/test/lit.site.cfg.py.in
@@ -15,6 +15,8 @@ config.native_target = "@LLVM_NATIVE_ARCH@"
config.host_os = "@HOST_OS@"
config.host_cc = "@HOST_CC@"
config.host_cxx = "@HOST_CXX@"
+config.host_c_compiler_launcher = "@CMAKE_C_COMPILER_LAUNCHER@"
+config.host_cxx_compiler_launcher = "@CMAKE_CXX_COMPILER_LAUNCHER@"
config.enable_libcxx = "@LLVM_ENABLE_LIBCXX@"
config.host_cmake = "@CMAKE_COMMAND@"
config.host_cmake_generator = "@CMAKE_GENERATOR@"
|
makslevental
changed the title
makslevental
force-pushed
the
users/makslevental/standalone-whee
branch
from
5ea0008 to
9fcecd4
Compare
makslevental
force-pushed
the
users/makslevental/standalone-whee
branch
2 times, most recently
from
441dc6c to
f6546b3
Compare
jpienaar
approved these changes
Sep 24, 2025
| CMAKE_CXX_COMPILER_LAUNCHER = { env = "CMAKE_CXX_COMPILER_LAUNCHER", default = "" } | ||
| CMAKE_GENERATOR = { env = "CMAKE_GENERATOR", default = "Ninja" } | ||
| LLVM_USE_LINKER = { env = "LLVM_USE_LINKER", default = "" } | ||
| # Optional but highly recommended. |
There was a problem hiding this comment.
Could you expand this comment? (our cmake is a mysterious land of tribal knowledge, so good to make reasoning easier to discover :))
| # RUN: export LLVM_USE_LINKER=%llvm_use_linker | ||
| # RUN: export MLIR_DIR="%mlir_cmake_dir" | ||
|
|
||
| # RUN: %python -m pip uninstall -y standalone-python-bindings |
There was a problem hiding this comment.
Should we be doing this in an environment? (e.g., could this have non-test local effects?)
There was a problem hiding this comment.
I tried but the GH runners don't have python3-venv installed so it's not possible. Whether there could be side-effects (no pun intended) I think it's unlikely that someone at some point actually has a useful package called standalone-python-bindings but I could name mangle it more.
There was a problem hiding this comment.
Actually nm I know what to do - you can do pip install --target SOMEWHERE to install to an arbitrary location.
There was a problem hiding this comment.
all cleaned up
There was a problem hiding this comment.
So this becomes the slowest test in the test suite. But only enabled if example and python is?
45.20s: MLIR :: Examples/standalone/test.wheel.toy
15.08s: MLIR :: Examples/standalone/test.toy
...
There was a problem hiding this comment.
Correct. Also it's important to note that this isn't wall-clock time - the tests run in parallel. So here (ie GH pre-merge CI) there's basically no difference.
makslevental
force-pushed
the
users/makslevental/standalone-whee
branch
3 times, most recently
from
dc4a9ce to
4731cfd
Compare
makslevental
force-pushed
the
users/makslevental/standalone-whee
branch
from
4731cfd to
b05e8e1
Compare
|
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/138/builds/19507 Here is the relevant piece of the build log for the reference |
makslevental
added a commit
that referenced
this pull request
Sep 24, 2025
This reverts commit 1359f3a.
makslevental
added a commit
that referenced
this pull request
Sep 24, 2025
Reverts #160388 because it broke [mlir-nvidia](https://lab.llvm.org/buildbot/#/builders/138) builder.
llvm-sync bot
pushed a commit
to arm/arm-toolchain
that referenced
this pull request
Sep 24, 2025
…#160481) Reverts llvm/llvm-project#160388 because it broke [mlir-nvidia](https://lab.llvm.org/buildbot/#/builders/138) builder.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
4 participants
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This PR demos and tests building Python wheels using scikit-build-core. The test is added to standalone and thus demos "out-of-tree" use cases but the same
pyproject.tomlwill work for in-tree builds. Note, one can easily pair this with cibuildwheel to build for all Python versions, OSs, architectures, etc.