Switch to scikit-build-core (#13531)

Contributes to rapidsai/build-planning#2

Authors:
  - Vyas Ramasubramani (https://github.com/vyasr)

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - AJ Schmidt (https://github.com/ajschmidt8)
  - Richard (Rick) Zamora (https://github.com/rjzamora)

URL: #13531

build.sh

@@ -332,8 +332,7 @@ fi
 if buildAll || hasArg cudf; then
 
     cd ${REPODIR}/python/cudf
-    SKBUILD_CONFIGURE_OPTIONS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR} -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} ${EXTRA_CMAKE_ARGS}" \
-        SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL:-1}" \
+    SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR};-DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES};${EXTRA_CMAKE_ARGS}" \
         python -m pip install --no-build-isolation --no-deps .
 fi
 
@@ -369,15 +368,12 @@ fi
 # build cudf_kafka Python package
 if hasArg cudf_kafka; then
     cd ${REPODIR}/python/cudf_kafka
-    SKBUILD_CONFIGURE_OPTIONS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR} ${EXTRA_CMAKE_ARGS}" \
-        SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL:-1}" \
+    SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR};${EXTRA_CMAKE_ARGS}"
         python -m pip install --no-build-isolation --no-deps .
 fi
 
 # build custreamz Python package
 if hasArg custreamz; then
     cd ${REPODIR}/python/custreamz
-    SKBUILD_CONFIGURE_OPTIONS="-DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR}" \
-        SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL:-1}" \
-        python -m pip install --no-build-isolation --no-deps .
+    python -m pip install --no-build-isolation --no-deps .
 fi

ci/cudf_pandas_scripts/pandas-tests/run.sh

@@ -23,7 +23,7 @@ manylinux="manylinux_${manylinux_version}"
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep
-python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,pandas_tests]
+python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,pandas-tests]
 
 git checkout $COMMIT
 

ci/cudf_pandas_scripts/run_tests.sh

@@ -46,7 +46,7 @@ else
 
     RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
     RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep
-    python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,cudf_pandas_tests]
+    python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,cudf-pandas-tests]
 fi
 
 python -m pytest -p cudf.pandas ./python/cudf/cudf_pandas_tests/

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -86,7 +86,7 @@ dependencies:
 - rich
 - rmm==24.2.*
 - s3fs>=2022.3.0
-- scikit-build>=0.13.1
+- scikit-build-core>=0.7.0
 - scipy
 - spdlog>=1.12.0,<1.13
 - sphinx

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -83,7 +83,7 @@ dependencies:
 - rich
 - rmm==24.2.*
 - s3fs>=2022.3.0
-- scikit-build>=0.13.1
+- scikit-build-core>=0.7.0
 - scipy
 - spdlog>=1.12.0,<1.13
 - sphinx

conda/recipes/cudf/meta.yaml

@@ -58,7 +58,7 @@ requirements:
     - protobuf ==4.24.*
     - python
     - cython >=3.0.3
-    - scikit-build >=0.13.1
+    - scikit-build-core >=0.7.0
     - setuptools
     - dlpack >=0.5,<0.6.0a0
     - pyarrow ==14.0.1.*

conda/recipes/cudf_kafka/meta.yaml

@@ -57,7 +57,7 @@ requirements:
     - cuda-version ={{ cuda_version }}
     - cudf ={{ version }}
     - libcudf_kafka ={{ version }}
-    - scikit-build >=0.13.1
+    - scikit-build-core >=0.7.0
     - setuptools
     {% if cuda_major == "12" %}
     - cuda-cudart-dev

cpp/cmake/thirdparty/get_arrow.cmake

@@ -77,6 +77,7 @@ list(POP_BACK CMAKE_PREFIX_PATH)
   # benchmarks will not work without updating GBench (and possibly NVBench) builds. We are currently
   # ignoring these limitations since we don't anticipate using this feature except for building
   # wheels.
+  enable_language(C)
   execute_process(
     COMMAND ${CMAKE_C_COMPILER} -print-file-name=libc.so.6
     OUTPUT_VARIABLE GLIBC_EXECUTABLE

dependencies.yaml

@@ -71,10 +71,8 @@ files:
     extras:
       table: build-system
     includes:
-      - build_all
       - build_python_common
       - build_python_cudf
-      - build_wheels
   py_run_cudf:
     output: pyproject
     pyproject_dir: python/cudf
@@ -99,15 +97,15 @@ files:
     pyproject_dir: python/cudf
     extras:
       table: project.optional-dependencies
-      key: pandas_tests
+      key: pandas-tests
     includes:
       - test_python_pandas_cudf
   py_test_cudf_pandas:
     output: pyproject
     pyproject_dir: python/cudf
     extras:
       table: project.optional-dependencies
-      key: cudf_pandas_tests
+      key: cudf-pandas-tests
     includes:
       - test_python_cudf_pandas
   py_build_dask_cudf:
@@ -143,7 +141,6 @@ files:
       table: build-system
     includes:
       - build_python_common
-      - build_wheels
   py_run_cudf_kafka:
     output: pyproject
     pyproject_dir: python/cudf_kafka
@@ -193,12 +190,10 @@ channels:
 dependencies:
   build_all:
     common:
-      - output_types: [conda, requirements, pyproject]
-        packages:
-          - &cmake_ver cmake>=3.26.4
-          - ninja
       - output_types: conda
         packages:
+          - &cmake_ver cmake>=3.26.4
+          - &ninja ninja
           - c-compiler
           - cxx-compiler
           - dlpack>=0.5,<0.6.0a0
@@ -257,15 +252,20 @@ dependencies:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
+          - *cmake_ver
           - cython>=3.0.3
+          - *ninja
           # TODO: Pin to numpy<1.25 until cudf requires pandas 2
           - &numpy numpy>=1.21,<1.25
-          - scikit-build>=0.13.1
-      - output_types: [conda, requirements, pyproject]
-        packages:
           # Hard pin the patch version used during the build. This must be kept
           # in sync with the version pinned in get_arrow.cmake.
           - pyarrow==14.0.1.*
+      - output_types: conda
+        packages:
+          - scikit-build-core>=0.7.0
+      - output_types: [requirements, pyproject]
+        packages:
+          - scikit-build-core[pyproject]>=0.7.0
   build_python_cudf:
     common:
       - output_types: conda

python/cudf/CMakeLists.txt

@@ -23,11 +23,7 @@ rapids_cuda_init_architectures(cudf-python)
 project(
   cudf-python
   VERSION ${cudf_version}
-  LANGUAGES # TODO: Building Python extension modules via the python_extension_module requires the C
-            # language to be enabled here. The test project that is built in scikit-build to verify
-            # various linking options for the python library is hardcoded to build with C, so until
-            # that is fixed we need to keep C.
-            C CXX CUDA
+  LANGUAGES CXX CUDA
 )
 
 option(FIND_CUDF_CPP "Search for existing CUDF C++ installations before defaulting to local files"
@@ -68,7 +64,7 @@ else()
   set(cudf_FOUND OFF)
 endif()
 
-include(rapids-cython)
+include(rapids-cython-core)
 
 if(NOT cudf_FOUND)
   set(BUILD_TESTS OFF)

python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake

@@ -13,22 +13,7 @@
 # =============================================================================
 include_guard(GLOBAL)
 
-# TODO: Finding NumPy currently requires finding Development due to a bug in CMake. This bug was
-# fixed in https://gitlab.kitware.com/cmake/cmake/-/merge_requests/7410 and will be available in
-# CMake 3.24, so we can remove the Development component once we upgrade to CMake 3.24.
-# find_package(Python REQUIRED COMPONENTS Development NumPy)
-
-# Note: The bug noted above prevents us from finding NumPy successfully using FindPython.cmake
-# inside the manylinux images used to build wheels because manylinux images do not contain
-# libpython.so and therefore Development cannot be found. Until we upgrade to CMake 3.24, we should
-# use FindNumpy.cmake instead (provided by scikit-build). When we switch to 3.24 we can try
-# switching back, but it may not work if that implicitly still requires Python libraries. In that
-# case we'll need to follow up with the CMake team to remove that dependency.  The stopgap solution
-# is to unpack the static lib tarballs in the wheel building jobs so that there are at least static
-# libs to be found, but that should be a last resort since it implies a dependency that isn't really
-# necessary. The relevant command is tar -xf /opt/_internal/static-libs-for-embedding-only.tar.xz -C
-# /opt/_internal"
-find_package(NumPy REQUIRED)
+find_package(Python REQUIRED COMPONENTS Development NumPy)
 
 execute_process(
   COMMAND "${Python_EXECUTABLE}" -c "import pyarrow; print(pyarrow.get_include())"
@@ -49,7 +34,7 @@ endif()
 function(link_to_pyarrow_headers targets)
   foreach(target IN LISTS targets)
     # PyArrow headers require numpy headers.
-    target_include_directories(${target} PRIVATE "${NumPy_INCLUDE_DIRS}")
+    target_include_directories(${target} PRIVATE "${Python_NumPy_INCLUDE_DIRS}")
     target_include_directories(${target} PRIVATE "${PYARROW_INCLUDE_DIR}")
   endforeach()
 endfunction()

python/cudf/cudf/_lib/CMakeLists.txt

@@ -63,7 +63,7 @@ rapids_cython_create_modules(
   LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cudf
 )
 
-target_link_libraries(strings_udf cudf_strings_udf)
+target_link_libraries(strings_udf PUBLIC cudf_strings_udf)
 
 link_to_pyarrow_headers("${RAPIDS_CYTHON_CREATED_TARGETS}")
 

python/cudf/cudf/_lib/io/CMakeLists.txt

@@ -1,5 +1,5 @@
 # =============================================================================
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at
@@ -20,8 +20,4 @@ rapids_cython_create_modules(
   LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX io_ ASSOCIATED_TARGETS cudf
 )
 
-set(targets_using_numpy_and_arrow_headers io_datasource io_utils)
-foreach(target IN LISTS targets_using_numpy_and_arrow_headers)
-  target_include_directories(${target} PRIVATE "${NumPy_INCLUDE_DIRS}")
-  target_include_directories(${target} PRIVATE "${PYARROW_INCLUDE_DIR}")
-endforeach()
+link_to_pyarrow_headers("${RAPIDS_CYTHON_CREATED_TARGETS}")

python/cudf/pyproject.toml

@@ -1,7 +1,7 @@
 # Copyright (c) 2021-2023, NVIDIA CORPORATION.
 
 [build-system]
-build-backend = "setuptools.build_meta"
+build-backend = "scikit_build_core.build"
 requires = [
     "cmake>=3.26.4",
     "cython>=3.0.3",
@@ -10,9 +10,7 @@ requires = [
     "protoc-wheel",
     "pyarrow==14.0.1.*",
     "rmm==24.2.*",
-    "scikit-build>=0.13.1",
-    "setuptools",
-    "wheel",
+    "scikit-build-core[pyproject]>=0.7.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project]
@@ -71,7 +69,7 @@ test = [
     "transformers==4.24.0",
     "tzdata",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
-pandas_tests = [
+pandas-tests = [
     "beautifulsoup4",
     "blosc",
     "boto3",
@@ -114,7 +112,7 @@ pandas_tests = [
     "xlwt",
     "zstandard",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
-cudf_pandas_tests = [
+cudf-pandas-tests = [
     "ipython",
     "openpyxl",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
@@ -123,12 +121,6 @@ cudf_pandas_tests = [
 Homepage = "https://github.com/rapidsai/cudf"
 Documentation = "https://docs.rapids.ai/api/cudf/stable/"
 
-[tool.setuptools]
-license-files = ["LICENSE"]
-
-[tool.setuptools.dynamic]
-version = {file = "cudf/VERSION"}
-
 [tool.isort]
 line_length = 79
 multi_line_output = 3
@@ -171,3 +163,16 @@ skip = [
     "dist",
     "__init__.py",
 ]
+
+[tool.scikit-build]
+build-dir = "build/{wheel_tag}"
+cmake.build-type = "Release"
+cmake.minimum-version = "3.26.4"
+ninja.make-fallback = true
+sdist.reproducible = true
+wheel.packages = ["cudf"]
+
+[tool.scikit-build.metadata.version]
+provider = "scikit_build_core.metadata.regex"
+input = "cudf/VERSION"
+regex = "(?P<value>.*)"

python/cudf_kafka/CMakeLists.txt

@@ -21,11 +21,7 @@ include(../../fetch_rapids.cmake)
 project(
   cudf-kafka-python
   VERSION ${cudf_kafka_version}
-  LANGUAGES # TODO: Building Python extension modules via the python_extension_module requires the C
-            # language to be enabled here. The test project that is built in scikit-build to verify
-            # various linking options for the python library is hardcoded to build with C, so until
-            # that is fixed we need to keep C.
-            C CXX
+  LANGUAGES CXX
 )
 
 find_package(cudf_kafka ${cudf_kafka_version} REQUIRED)
@@ -37,7 +33,7 @@ if(NOT cudf_kafka_FOUND)
   )
 endif()
 
-include(rapids-cython)
+include(rapids-cython-core)
 rapids_cython_init()
 
 add_subdirectory(cudf_kafka/_lib)