[python-package] [ci] switch to PEP 517 / 518 builds (remove `setup.p…

…y`) (fixes #5061) (#5759)
microsoft · Jun 15, 2023 · 30942a3 · 30942a3
1 parent a323c75
commit 30942a3
Show file tree

Hide file tree

Showing 10 changed files with 224 additions and 491 deletions.
diff --git a/.ci/test.sh b/.ci/test.sh
@@ -159,6 +159,9 @@ elif [[ $TASK == "bdist" ]]; then
         sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
         mv \
             ./dist/*.whl \
+            ./dist/tmp.whl || exit -1
+        mv \
+            ./dist/tmp.whl \
             dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl || exit -1
         if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
             cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit -1
@@ -173,6 +176,9 @@ elif [[ $TASK == "bdist" ]]; then
         cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --integrated-opencl || exit -1
         mv \
             ./dist/*.whl \
+            ./dist/tmp.whl || exit -1
+        mv \
+            ./dist/tmp.whl \
             ./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit -1
         sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
         if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
@@ -186,12 +192,6 @@ elif [[ $TASK == "bdist" ]]; then
     exit 0
 fi
 
-# temporarily pin pip to versions that support 'pip install --install-option'
-# ref: https://github.com/microsoft/LightGBM/issues/5061#issuecomment-1510642287
-if [[ $METHOD == "pip" ]]; then
-    pip install 'pip<23.1'
-fi
-
 if [[ $TASK == "gpu" ]]; then
     sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
     grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
@@ -201,7 +201,7 @@ if [[ $TASK == "gpu" ]]; then
         pip install \
             --user \
             -v \
-            --install-option=--gpu \
+            --config-settings=cmake.define.USE_GPU=ON \
             $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
         || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
@@ -229,7 +229,7 @@ elif [[ $TASK == "cuda" ]]; then
         pip install \
             --user \
             -v \
-            --install-option=--cuda \
+            --config-settings=cmake.define.USE_CUDA=ON \
             $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
         || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
@@ -252,7 +252,7 @@ elif [[ $TASK == "mpi" ]]; then
         pip install \
             --user \
             -v \
-            --install-option=--mpi \
+            --config-settings=cmake.define.USE_MPI=ON \
             $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
         || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1

diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1
@@ -96,8 +96,8 @@ elseif ($env:TASK -eq "bdist") {
   cd $env:BUILD_SOURCESDIRECTORY
   sh "build-python.sh" bdist_wheel --integrated-opencl ; Check-Output $?
   sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/dist ; Check-Output $?
-  cd dist; pip install --user @(Get-ChildItem *.whl) ; Check-Output $?
-  cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
+  cd dist; pip install --user @(Get-ChildItem *py3-none-win_amd64.whl) ; Check-Output $?
+  cp @(Get-ChildItem *py3-none-win_amd64.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
 } elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
   cd $env:BUILD_SOURCESDIRECTORY
   if ($env:COMPILER -eq "MINGW") {

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -645,6 +645,10 @@ if(BUILD_CLI)
     )
 endif()
 
+if(__BUILD_FOR_PYTHON)
+    set(CMAKE_INSTALL_PREFIX "lightgbm")
+endif()
+
 install(
   TARGETS _lightgbm
   RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin

diff --git a/build-python.sh b/build-python.sh
@@ -94,75 +94,79 @@ while [ $# -gt 0 ]; do
             then shift;
         fi
         BOOST_DIR="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --boost-dir='${BOOST_DIR}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_DIR='${BOOST_DIR}'"
         ;;
     --boost-include-dir|--boost-include-dir=*)
         if [[ "$1" != *=* ]];
             then shift;
         fi
         BOOST_INCLUDE_DIR="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --boost-include-dir='${BOOST_INCLUDE_DIR}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_INCLUDE_DIR='${BOOST_INCLUDE_DIR}'"
         ;;
     --boost-librarydir|--boost-librarydir=*)
         if [[ "$1" != *=* ]];
             then shift;
         fi
         BOOST_LIBRARY_DIR="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --boost-librarydir='${BOOST_LIBRARY_DIR}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.BOOST_LIBRARYDIR='${BOOST_LIBRARY_DIR}'"
         ;;
     --boost-root|--boost-root=*)
         if [[ "$1" != *=* ]];
             then shift;
         fi
         BOOST_ROOT="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --boost-root='${BOOST_ROOT}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_ROOT='${BOOST_ROOT}'"
         ;;
     --opencl-include-dir|--opencl-include-dir=*)
         if [[ "$1" != *=* ]];
             then shift;
         fi
         OPENCL_INCLUDE_DIR="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --opencl-include-dir='${OPENCL_INCLUDE_DIR}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.OpenCL_INCLUDE_DIR='${OPENCL_INCLUDE_DIR}'"
         ;;
     --opencl-library|--opencl-library=*)
         if [[ "$1" != *=* ]];
             then shift;
         fi
         OPENCL_LIBRARY="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --opencl-library='${OPENCL_LIBRARY}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.OpenCL_LIBRARY='${OPENCL_LIBRARY}'"
         ;;
     #########
     # flags #
     #########
     --bit32)
-        BUILD_ARGS="${BUILD_ARGS} --bit32"
+        export CMAKE_GENERATOR="Visual Studio 17 2022"
+        export CMAKE_GENERATOR_PLATFORM="Win32"
+        echo "[INFO] Attempting to build 32-bit version of LightGBM, which is only supported on Windows with generator '${CMAKE_GENERATOR}'."
         ;;
     --cuda)
-        BUILD_ARGS="${BUILD_ARGS} --cuda"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_CUDA=ON"
         ;;
     --gpu)
-        BUILD_ARGS="${BUILD_ARGS} --gpu"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_GPU=ON"
         ;;
     --hdfs)
-        BUILD_ARGS="${BUILD_ARGS} --hdfs"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_HDFS=ON"
         ;;
     --integrated-opencl)
-        BUILD_ARGS="${BUILD_ARGS} --integrated-opencl"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.__INTEGRATE_OPENCL=ON"
         ;;
     --mingw)
-        BUILD_ARGS="${BUILD_ARGS} --mingw"
+        export CMAKE_GENERATOR='MinGW Makefiles'
+        # ref: https://stackoverflow.com/a/45104058/3986677
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.CMAKE_SH=CMAKE_SH-NOTFOUND"
         ;;
     --mpi)
-        BUILD_ARGS="${BUILD_ARGS} --mpi"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_MPI=ON"
         ;;
     --nomp)
-        BUILD_ARGS="${BUILD_ARGS} --nomp"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_OPENMP=OFF"
         ;;
     --precompile)
         PRECOMPILE="true"
         ;;
     --time-costs)
-        BUILD_ARGS="${PIP_INSTALL_ARGS} --time-costs"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_TIMETAG=ON"
         ;;
     --user)
         PIP_INSTALL_ARGS="${PIP_INSTALL_ARGS} --user"
@@ -175,6 +179,8 @@ while [ $# -gt 0 ]; do
   shift
 done
 
+pip install --prefer-binary 'build>=0.10.0'
+
 # create a new directory that just contains the files needed
 # to build the Python package
 create_isolated_source_dir() {
@@ -189,21 +195,15 @@ create_isolated_source_dir() {
 
     cp -R ./python-package ./lightgbm-python
 
-    # temporarily remove these files until
-    # https://github.com/microsoft/LightGBM/issues/5061 is done
-    rm ./lightgbm-python/pyproject.toml
-    rm ./lightgbm-python/setup.cfg
-
     cp LICENSE ./lightgbm-python/
     cp VERSION.txt ./lightgbm-python/lightgbm/VERSION.txt
 
-    mkdir -p ./lightgbm-python/compile
-    cp -R ./cmake ./lightgbm-python/compile
-    cp CMakeLists.txt ./lightgbm-python/compile
-    cp -R ./include ./lightgbm-python/compile
-    cp -R ./src ./lightgbm-python/compile
-    cp -R ./swig ./lightgbm-python/compile
-    cp -R ./windows ./lightgbm-python/compile
+    cp -R ./cmake ./lightgbm-python
+    cp CMakeLists.txt ./lightgbm-python
+    cp -R ./include ./lightgbm-python
+    cp -R ./src ./lightgbm-python
+    cp -R ./swig ./lightgbm-python
+    cp -R ./windows ./lightgbm-python
 
     # include only specific files from external_libs, to keep the package
     # small and avoid redistributing code with licenses incompatible with
@@ -212,77 +212,77 @@ create_isolated_source_dir() {
     ######################
     # fast_double_parser #
     ######################
-    mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser
+    mkdir -p ./lightgbm-python/external_libs/fast_double_parser
     cp \
         external_libs/fast_double_parser/CMakeLists.txt \
-        ./lightgbm-python/compile/external_libs/fast_double_parser/CMakeLists.txt
+        ./lightgbm-python/external_libs/fast_double_parser/CMakeLists.txt
     cp \
         external_libs/fast_double_parser/LICENSE* \
-        ./lightgbm-python/compile/external_libs/fast_double_parser/
+        ./lightgbm-python/external_libs/fast_double_parser/
 
-    mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser/include/
+    mkdir -p ./lightgbm-python/external_libs/fast_double_parser/include/
     cp \
         external_libs/fast_double_parser/include/fast_double_parser.h \
-        ./lightgbm-python/compile/external_libs/fast_double_parser/include/
+        ./lightgbm-python/external_libs/fast_double_parser/include/
 
     #######
     # fmt #
     #######
-    mkdir -p ./lightgbm-python/compile/external_libs/fmt
+    mkdir -p ./lightgbm-python/external_libs/fmt
     cp \
         external_libs/fast_double_parser/CMakeLists.txt \
-        ./lightgbm-python/compile/external_libs/fmt/CMakeLists.txt
+        ./lightgbm-python/external_libs/fmt/CMakeLists.txt
     cp \
         external_libs/fmt/LICENSE* \
-        ./lightgbm-python/compile/external_libs/fmt/
+        ./lightgbm-python/external_libs/fmt/
 
-    mkdir -p ./lightgbm-python/compile/external_libs/fmt/include/fmt
+    mkdir -p ./lightgbm-python/external_libs/fmt/include/fmt
     cp \
         external_libs/fmt/include/fmt/*.h \
-        ./lightgbm-python/compile/external_libs/fmt/include/fmt/
+        ./lightgbm-python/external_libs/fmt/include/fmt/
 
     #########
     # Eigen #
     #########
-    mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen
+    mkdir -p ./lightgbm-python/external_libs/eigen/Eigen
     cp \
         external_libs/eigen/CMakeLists.txt \
-        ./lightgbm-python/compile/external_libs/eigen/CMakeLists.txt
+        ./lightgbm-python/external_libs/eigen/CMakeLists.txt
 
     modules="Cholesky Core Dense Eigenvalues Geometry Householder Jacobi LU QR SVD"
     for eigen_module in ${modules}; do
         cp \
             external_libs/eigen/Eigen/${eigen_module} \
-            ./lightgbm-python/compile/external_libs/eigen/Eigen/${eigen_module}
+            ./lightgbm-python/external_libs/eigen/Eigen/${eigen_module}
         if [ ${eigen_module} != "Dense" ]; then
-            mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/
+            mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/src/${eigen_module}/
             cp \
                 -R \
                 external_libs/eigen/Eigen/src/${eigen_module}/* \
-                ./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/
+                ./lightgbm-python/external_libs/eigen/Eigen/src/${eigen_module}/
         fi
     done
 
-    mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/misc
+    mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/misc
     cp \
         -R \
         external_libs/eigen/Eigen/src/misc \
-        ./lightgbm-python/compile/external_libs/eigen/Eigen/src/misc/
+        ./lightgbm-python/external_libs/eigen/Eigen/src/misc/
 
-    mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/plugins
+    mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/plugins
     cp \
         -R \
         external_libs/eigen/Eigen/src/plugins \
-        ./lightgbm-python/compile/external_libs/eigen/Eigen/src/plugins/
+        ./lightgbm-python/external_libs/eigen/Eigen/src/plugins/
 
     ###################
     # compute (Boost) #
     ###################
-    mkdir -p ./lightgbm-python/compile/external_libs/compute
+    mkdir -p ./lightgbm-python/external_libs/compute
     cp \
         -R \
         external_libs/compute/include \
-        ./lightgbm-python/compile/external_libs/compute/include/
+        ./lightgbm-python/external_libs/compute/include/
 }
 
 create_isolated_source_dir
@@ -292,9 +292,39 @@ cd ./lightgbm-python
 # installation involves building the wheel + `pip install`-ing it
 if test "${INSTALL}" = true; then
     if test "${PRECOMPILE}" = true; then
-        echo "--- installing lightgbm (from precompiled lib_lightgbm) ---"
-        python setup.py install ${PIP_INSTALL_ARGS} --precompile
-        exit 0
+        BUILD_SDIST=true
+        BUILD_WHEEL=false
+        BUILD_ARGS=""
+        rm -rf \
+            ./cmake \
+            ./CMakeLists.txt \
+            ./external_libs \
+            ./include \
+            ./src \
+            ./swig \
+            ./windows
+        # use regular-old setuptools for these builds, to avoid
+        # trying to recompile the shared library
+        sed -i.bak -e '/start:build-system/,/end:build-system/d' pyproject.toml
+        echo '[build-system]' >> ./pyproject.toml
+        echo 'requires = ["setuptools"]' >> ./pyproject.toml
+        echo 'build-backend = "setuptools.build_meta"' >> ./pyproject.toml
+        echo "" >> ./pyproject.toml
+        echo "recursive-include lightgbm *.dll *.so" > ./MANIFEST.in
+        echo "" >> ./MANIFEST.in
+        mkdir -p ./lightgbm/lib
+        if test -f ../lib_lightgbm.so; then
+            echo "found pre-compiled lib_lightgbm.so"
+            cp ../lib_lightgbm.so ./lightgbm/lib/lib_lightgbm.so
+        elif test -f ../Release/lib_lightgbm.dll; then
+            echo "found pre-compiled Release/lib_lightgbm.dll"
+            cp ../Release/lib_lightgbm.dll ./lightgbm/lib/lib_lightgbm.dll
+        elif test -f ../windows/x64/DLL/lib_lightgbm.dll; then
+            echo "found pre-compiled windows/x64/DLL/lib_lightgbm.dll"
+            cp ../windows/x64/DLL/lib_lightgbm.dll ./lightgbm/lib/lib_lightgbm.dll
+            cp ../windows/x64/DLL/lib_lightgbm.lib ./lightgbm/lib/lib_lightgbm.lib
+        fi
+        rm -f ./*.bak
     else
         BUILD_SDIST="false"
         BUILD_WHEEL="true"
@@ -304,16 +334,20 @@ fi
 if test "${BUILD_SDIST}" = true; then
     echo "--- building sdist ---"
     rm -f ../dist/*.tar.gz
-    python ./setup.py sdist \
-        --dist-dir ../dist
+    python -m build \
+        --sdist \
+        --outdir ../dist \
+        .
 fi
 
 if test "${BUILD_WHEEL}" = true; then
     echo "--- building wheel ---"
     rm -f ../dist/*.whl || true
-    python setup.py bdist_wheel \
-        --dist-dir ../dist \
-        ${BUILD_ARGS}
+    python -m build \
+        --wheel \
+        --outdir ../dist \
+        ${BUILD_ARGS} \
+        .
 fi
 
 if test "${INSTALL}" = true; then