From 9c930283e35213874108882e8fac72e5e6cd0a59 Mon Sep 17 00:00:00 2001 From: makslevental Date: Wed, 24 Sep 2025 03:11:35 -0700 Subject: [PATCH] Reland "[MLIR][Python] add Python wheel build demo/test" (#160481) --- mlir/examples/standalone/CMakeLists.txt | 8 ++- mlir/examples/standalone/pyproject.toml | 65 ++++++++++++++++++++ mlir/test/CMakeLists.txt | 1 + mlir/test/Examples/standalone/lit.local.cfg | 6 ++ mlir/test/Examples/standalone/test.wheel.toy | 32 ++++++++++ mlir/test/lit.site.cfg.py.in | 3 + 6 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 mlir/examples/standalone/pyproject.toml create mode 100644 mlir/test/Examples/standalone/test.wheel.toy diff --git a/mlir/examples/standalone/CMakeLists.txt b/mlir/examples/standalone/CMakeLists.txt index e2bcda7fa6f0b..c6c49fde12d2e 100644 --- a/mlir/examples/standalone/CMakeLists.txt +++ b/mlir/examples/standalone/CMakeLists.txt @@ -63,8 +63,12 @@ if(MLIR_ENABLE_BINDINGS_PYTHON) include(MLIRDetectPythonEnv) mlir_configure_python_dev_packages() # Note: for EXTERNAL_PROJECT_BUILD this must be set from the command line. - set(MLIR_PYTHON_PACKAGE_PREFIX "mlir_standalone" CACHE STRING "" FORCE) - set(MLIR_BINDINGS_PYTHON_INSTALL_PREFIX "python_packages/standalone/${MLIR_PYTHON_PACKAGE_PREFIX}" CACHE STRING "" FORCE) + if(NOT MLIR_PYTHON_PACKAGE_PREFIX) + set(MLIR_PYTHON_PACKAGE_PREFIX "mlir_standalone" CACHE STRING "" FORCE) + endif() + if(NOT MLIR_BINDINGS_PYTHON_INSTALL_PREFIX) + set(MLIR_BINDINGS_PYTHON_INSTALL_PREFIX "python_packages/standalone/${MLIR_PYTHON_PACKAGE_PREFIX}" CACHE STRING "" FORCE) + endif() add_subdirectory(python) endif() add_subdirectory(test) diff --git a/mlir/examples/standalone/pyproject.toml b/mlir/examples/standalone/pyproject.toml new file mode 100644 index 0000000000000..5a1e6e86513c3 --- /dev/null +++ b/mlir/examples/standalone/pyproject.toml @@ -0,0 +1,65 @@ +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# Copyright (c) 2025. + +[project] +name = "standalone-python-bindings" +dynamic = ["version"] +requires-python = ">=3.8,<=3.14" +dependencies = [ + "numpy>=1.19.5, <=2.1.2", + "PyYAML>=5.4.0, <=6.0.1", + "ml_dtypes>=0.1.0, <=0.6.0; python_version<'3.13'", + "ml_dtypes>=0.5.0, <=0.6.0; python_version>='3.13'", +] + +[project.urls] +Homepage = "https://github.com/llvm/llvm-project" +Discussions = "https://discourse.llvm.org/" +"Issue Tracker" = "https://github.com/llvm/llvm-project/issues?q=is%3Aissue%20state%3Aopen%20label%3Amlir%3Apython%20" +"Source Code" = "https://github.com/llvm/llvm-project/tree/main/mlir/python" + +[build-system] +requires = [ + "scikit-build-core>=0.10.7", + "typing_extensions>=4.12.2", + "nanobind>=2.9, <3.0", + "pybind11>=2.10.0, <=2.13.6", +] +build-backend = "scikit_build_core.build" + +[tool.scikit-build] +# This is the minimum version of scikit-build-core. +minimum-version = "0.10.7" +# This pyproject.toml must be adjacent to the root CMakeLists.txt (wherever project(...) is specified). +cmake.source-dir = "." +# This is for installing/distributing the python bindings target and only the python bindings target. +build.targets = ["StandalonePythonModules"] +install.components = ["StandalonePythonModules"] + +[tool.scikit-build.cmake.define] +# Optional +CMAKE_C_COMPILER = { env = "CMAKE_C_COMPILER", default = "" } +CMAKE_CXX_COMPILER = { env = "CMAKE_CXX_COMPILER", default = "" } +CMAKE_C_COMPILER_LAUNCHER = { env = "CMAKE_C_COMPILER_LAUNCHER", default = "" } +CMAKE_CXX_COMPILER_LAUNCHER = { env = "CMAKE_CXX_COMPILER_LAUNCHER", default = "" } +CMAKE_GENERATOR = { env = "CMAKE_GENERATOR", default = "Ninja" } +LLVM_USE_LINKER = { env = "LLVM_USE_LINKER", default = "" } +# Optional but highly recommended (this makes the bindings compatible with other bindings packages +# by preventing symbol collisions). +CMAKE_VISIBILITY_INLINES_HIDDEN = "ON" +CMAKE_C_VISIBILITY_PRESET = "hidden" +CMAKE_CXX_VISIBILITY_PRESET = "hidden" + +# Non-optional (alternatively you could use CMAKE_PREFIX_PATH here). +MLIR_DIR = { env = "MLIR_DIR", default = "" } +# Non-optional +CMAKE_BUILD_TYPE = { env = "CMAKE_BUILD_TYPE", default = "Release" } +MLIR_ENABLE_BINDINGS_PYTHON = "ON" +# Effectively non-optional (any downstream project should specify this). +MLIR_PYTHON_PACKAGE_PREFIX = "mlir_standalone" +# This specifies the directory in the install directory (i.e., /tmp/pip-wheel/platlib) where _mlir_libs, dialects, etc. +# are installed. Thus, this will be the package location (and the name of the package) that pip assumes is +# the root package. +MLIR_BINDINGS_PYTHON_INSTALL_PREFIX = "mlir_standalone" diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt index 628adcfb6e285..98b7b71b660ec 100644 --- a/mlir/test/CMakeLists.txt +++ b/mlir/test/CMakeLists.txt @@ -84,6 +84,7 @@ llvm_canonicalize_cmake_booleans( MLIR_RUN_CUDA_SM80_TESTS MLIR_RUN_CUDA_SM80_LT_TESTS MLIR_RUN_CUDA_SM90_TESTS + BUILD_SHARED_LIBS ) configure_lit_site_cfg( diff --git a/mlir/test/Examples/standalone/lit.local.cfg b/mlir/test/Examples/standalone/lit.local.cfg index 3b12dcbd99e83..ac03503e46ea3 100644 --- a/mlir/test/Examples/standalone/lit.local.cfg +++ b/mlir/test/Examples/standalone/lit.local.cfg @@ -7,7 +7,13 @@ config.substitutions.append(("%cmake_exe", config.host_cmake)) config.substitutions.append(("%cmake_generator", config.host_cmake_generator)) config.substitutions.append(("%host_cxx", config.host_cxx)) config.substitutions.append(("%host_cc", config.host_cc)) +config.substitutions.append(("%hostc_compiler_launcher", config.host_c_compiler_launcher)) +config.substitutions.append(("%hostcxx_compiler_launcher", config.host_cxx_compiler_launcher)) config.substitutions.append(("%enable_libcxx", config.enable_libcxx)) config.substitutions.append(("%mlir_cmake_dir", config.mlir_cmake_dir)) +config.substitutions.append(("%mlir_obj_root", config.mlir_obj_root)) config.substitutions.append(("%llvm_use_linker", config.llvm_use_linker)) config.substitutions.append(("%cmake_build_type", config.cmake_build_type)) + +if not config.llvm_shared_libs_build: + config.available_features.add("non-shared-libs-build") diff --git a/mlir/test/Examples/standalone/test.wheel.toy b/mlir/test/Examples/standalone/test.wheel.toy new file mode 100644 index 0000000000000..17d8cb5b246c9 --- /dev/null +++ b/mlir/test/Examples/standalone/test.wheel.toy @@ -0,0 +1,32 @@ +# There's no real issue with windows here, it's just that some CMake generated paths for targets end up being longer +# than 255 chars when combined with the fact that pip wants to install into a tmp directory buried under +# C/Users/ContainerAdministrator/AppData/Local/Temp. +# UNSUPPORTED: target={{.*(windows).*}} +# REQUIRES: non-shared-libs-build + +# RUN: export CMAKE_BUILD_TYPE=%cmake_build_type +# RUN: export CMAKE_CXX_COMPILER=%host_cxx +# RUN: export CMAKE_CXX_COMPILER_LAUNCHER=%hostcxx_compiler_launcher +# RUN: export CMAKE_C_COMPILER=%host_cc +# RUN: export CMAKE_C_COMPILER_LAUNCHER=%hostc_compiler_launcher +# RUN: export CMAKE_GENERATOR=%cmake_generator +# RUN: export LLVM_USE_LINKER=%llvm_use_linker +# RUN: export MLIR_DIR="%mlir_cmake_dir" + +# RUN: %python -m pip wheel "%mlir_src_root/examples/standalone" -w "%mlir_obj_root/wheelhouse" -v | tee %t + +# RUN: rm -rf "%mlir_obj_root/standalone-python-bindings-install" +# RUN: %python -m pip install standalone_python_bindings -f "%mlir_obj_root/wheelhouse" --target "%mlir_obj_root/standalone-python-bindings-install" -v | tee -a %t + +# RUN: export PYTHONPATH="%mlir_obj_root/standalone-python-bindings-install" +# RUN: %python "%mlir_src_root/examples/standalone/test/python/smoketest.py" nanobind | tee -a %t + +# RUN: FileCheck --input-file=%t %s + +# CHECK: Successfully built standalone-python-bindings + +# CHECK: module { +# CHECK: %[[C2:.*]] = arith.constant 2 : i32 +# CHECK: %[[V0:.*]] = standalone.foo %[[C2]] : i32 +# CHECK: } + diff --git a/mlir/test/lit.site.cfg.py.in b/mlir/test/lit.site.cfg.py.in index 2fc595dfabbf5..1aaf7989e3ce5 100644 --- a/mlir/test/lit.site.cfg.py.in +++ b/mlir/test/lit.site.cfg.py.in @@ -15,6 +15,8 @@ config.native_target = "@LLVM_NATIVE_ARCH@" config.host_os = "@HOST_OS@" config.host_cc = "@HOST_CC@" config.host_cxx = "@HOST_CXX@" +config.host_c_compiler_launcher = "@CMAKE_C_COMPILER_LAUNCHER@" +config.host_cxx_compiler_launcher = "@CMAKE_CXX_COMPILER_LAUNCHER@" config.enable_libcxx = "@LLVM_ENABLE_LIBCXX@" config.host_cmake = "@CMAKE_COMMAND@" config.host_cmake_generator = "@CMAKE_GENERATOR@" @@ -58,6 +60,7 @@ config.mlir_run_cuda_sm80_tests = @MLIR_RUN_CUDA_SM80_TESTS@ config.mlir_run_cuda_sm80_lt_tests = @MLIR_RUN_CUDA_SM80_LT_TESTS@ config.mlir_run_cuda_sm90_tests = @MLIR_RUN_CUDA_SM90_TESTS@ config.mlir_include_integration_tests = @MLIR_INCLUDE_INTEGRATION_TESTS@ +config.llvm_shared_libs_build = @BUILD_SHARED_LIBS@ config.arm_emulator_executable = "@ARM_EMULATOR_EXECUTABLE@" # Some tests marked with 'UNSUPPORTED: target=aarch64{{.*}}' are still run when # configured with ARM_EMULATOR_EXECUTABLE and the default target is not aarch64.