From f37477cb6b3642c5b74e1087bcacca51b1beff0f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 27 Oct 2023 12:09:18 -0700 Subject: [PATCH 01/20] Bump arrow pinning for wheels --- cpp/cmake/thirdparty/get_arrow.cmake | 2 +- dependencies.yaml | 4 ++-- python/cudf/pyproject.toml | 4 ++-- python/cudf_kafka/pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index 3b2cbc57d1c..0768ac30045 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -413,7 +413,7 @@ if(NOT DEFINED CUDF_VERSION_Arrow) set(CUDF_VERSION_Arrow # This version must be kept in sync with the libarrow version pinned for builds in # dependencies.yaml. - 12.0.1 + 13.0.0 CACHE STRING "The version of Arrow to find (or build)" ) else() diff --git a/dependencies.yaml b/dependencies.yaml index 1f2b42c49c4..2b7995a6672 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -251,7 +251,7 @@ dependencies: packages: # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - pyarrow==12.0.1.* + - pyarrow==13.0.0.* build_python: common: - output_types: [conda, requirements, pyproject] @@ -279,7 +279,7 @@ dependencies: - output_types: [requirements, pyproject] packages: # Allow runtime version to float up to minor version - - pyarrow==12.* + - pyarrow==13.* cudatoolkit: specific: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 90759074750..0be49dab4b3 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -8,7 +8,7 @@ requires = [ "ninja", "numpy>=1.21,<1.25", "protoc-wheel", - "pyarrow==12.0.1.*", + "pyarrow==13.0.0.*", "rmm==23.12.*", "scikit-build>=0.13.1", "setuptools", @@ -38,7 +38,7 @@ dependencies = [ "pandas>=1.3,<1.6.0dev0", "protobuf>=4.21,<5", "ptxcompiler", - "pyarrow==12.*", + "pyarrow==13.*", "rmm==23.12.*", "typing_extensions>=4.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 78a7a83ac3a..5058412ef60 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "cython>=3.0.0", "numpy>=1.21,<1.25", - "pyarrow==12.0.1.*", + "pyarrow==13.0.0.*", "setuptools", "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 50b148a5f8cdede8e608a3c0b5ddb398ed28adc7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 27 Oct 2023 14:16:59 -0700 Subject: [PATCH 02/20] Set CXX11 ABI flag conditionally based on the system glibc version. --- cpp/cmake/thirdparty/get_arrow.cmake | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index 0768ac30045..de4075e6a9f 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -62,10 +62,24 @@ function(find_libarrow_in_python_wheel PYARROW_VERSION) # to builds of GTest. Similarly, benchmarks will not work without updating GBench (and possibly # NVBench) builds. We are currently ignoring these limitations since we don't anticipate using # this feature except for building wheels. - target_compile_options( - Arrow::Arrow INTERFACE "$<$:-D_GLIBCXX_USE_CXX11_ABI=0>" - "$<$:-Xcompiler=-D_GLIBCXX_USE_CXX11_ABI=0>" - ) + EXECUTE_PROCESS ( + COMMAND ${CMAKE_C_COMPILER} -print-file-name=libc.so.6 + OUTPUT_VARIABLE GLIBC_EXECUTABLE + OUTPUT_STRIP_TRAILING_WHITESPACE) + EXECUTE_PROCESS ( + COMMAND ${GLIBC_EXECUTABLE} + OUTPUT_VARIABLE GLIBC_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE) + STRING(REGEX MATCH "stable release version ([0-9]+\\.[0-9]+)" GLIBC_VERSION ${GLIBC_OUTPUT}) + STRING(REPLACE "stable release version " "" GLIBC_VERSION ${GLIBC_VERSION}) + STRING(REPLACE "." ";" GLIBC_VERSION_LIST ${GLIBC_VERSION}) + LIST(GET GLIBC_VERSION_LIST 1 GLIBC_VERSION_MINOR) + if(GLIBC_VERSION_MINOR LESS 28) + target_compile_options( + Arrow::Arrow INTERFACE "$<$:-D_GLIBCXX_USE_CXX11_ABI=0>" + "$<$:-Xcompiler=-D_GLIBCXX_USE_CXX11_ABI=0>" + ) + endif() rapids_export_package(BUILD Arrow cudf-exports) rapids_export_package(INSTALL Arrow cudf-exports) From 2c3f79e1472d8555e6815d73095a329893f31223 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 27 Oct 2023 14:40:33 -0700 Subject: [PATCH 03/20] Fix format --- cpp/cmake/thirdparty/get_arrow.cmake | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index de4075e6a9f..ff34610a8e3 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -62,14 +62,16 @@ function(find_libarrow_in_python_wheel PYARROW_VERSION) # to builds of GTest. Similarly, benchmarks will not work without updating GBench (and possibly # NVBench) builds. We are currently ignoring these limitations since we don't anticipate using # this feature except for building wheels. - EXECUTE_PROCESS ( - COMMAND ${CMAKE_C_COMPILER} -print-file-name=libc.so.6 - OUTPUT_VARIABLE GLIBC_EXECUTABLE - OUTPUT_STRIP_TRAILING_WHITESPACE) - EXECUTE_PROCESS ( - COMMAND ${GLIBC_EXECUTABLE} - OUTPUT_VARIABLE GLIBC_OUTPUT - OUTPUT_STRIP_TRAILING_WHITESPACE) + EXECUTE_PROCESS( + COMMAND ${CMAKE_C_COMPILER} -print-file-name=libc.so.6 + OUTPUT_VARIABLE GLIBC_EXECUTABLE + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + EXECUTE_PROCESS( + COMMAND ${GLIBC_EXECUTABLE} + OUTPUT_VARIABLE GLIBC_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE + ) STRING(REGEX MATCH "stable release version ([0-9]+\\.[0-9]+)" GLIBC_VERSION ${GLIBC_OUTPUT}) STRING(REPLACE "stable release version " "" GLIBC_VERSION ${GLIBC_VERSION}) STRING(REPLACE "." ";" GLIBC_VERSION_LIST ${GLIBC_VERSION}) From d73bc867d5fb37a088c0f58e42644a0a3187d6b0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 27 Oct 2023 17:56:30 -0700 Subject: [PATCH 04/20] Switch wheel builds to the new workflow --- .github/workflows/pr.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8d6c471c912..0feadc21be7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -98,9 +98,10 @@ jobs: wheel-build-cudf: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@feat/2_28_wheels with: build_type: pull-request + build-2_28-wheels: "true" script: "ci/build_wheel_cudf.sh" wheel-tests-cudf: needs: wheel-build-cudf From 8d847167116cc1fe79bf2e166818b0ff38ddd72a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 28 Oct 2023 08:47:40 -0700 Subject: [PATCH 05/20] Install jq --- ci/build_wheel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 06d0c3c7a56..d786c6ae782 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -6,6 +6,7 @@ set -euo pipefail package_name=$1 package_dir=$2 +dnf install -y jq source rapids-configure-sccache source rapids-date-string From c6d2054b07e6ea9fe228790245687f3f4b55d0fc Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 28 Oct 2023 09:02:38 -0700 Subject: [PATCH 06/20] Only install on rocky --- ci/build_wheel.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index d786c6ae782..02b47327833 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -6,7 +6,9 @@ set -euo pipefail package_name=$1 package_dir=$2 -dnf install -y jq +if command -v dnf >/dev/null 2>&1 ; then + dnf install -y jq +fi source rapids-configure-sccache source rapids-date-string From d92161ec23012d7ee71b92ceae0c0e22d5a904d8 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 28 Oct 2023 10:42:27 -0700 Subject: [PATCH 07/20] Manually overwrite the manylinux policy for rocky --- ci/build_wheel_cudf.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 7d3919b2d72..1f6533917e1 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -10,6 +10,13 @@ export SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" ./ci/build_wheel.sh cudf ${package_dir} mkdir -p ${package_dir}/final_dist +if command -v dnf >/dev/null 2>&1 ; then + export POLICY="manylinux_2_28" + export REAL_ARCH=$(arch) + export AUDITWHEEL_POLICY=${POLICY} + export AUDITWHEEL_ARCH=${REAL_ARCH} + export AUDITWHEEL_PLAT=${POLICY}_${REAL_ARCH} +fi python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" From 626df446eeace712faec1f776a99b7d67917b0a3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 28 Oct 2023 13:24:53 -0700 Subject: [PATCH 08/20] Encode the manylinux version in the built wheels and pull appropriately --- ci/build_wheel_cudf.sh | 4 +++- ci/test_wheel_cudf.sh | 12 +++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 1f6533917e1..eefd2a07de0 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -9,6 +9,7 @@ export SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" ./ci/build_wheel.sh cudf ${package_dir} +manylinux="manylinux_2_17" mkdir -p ${package_dir}/final_dist if command -v dnf >/dev/null 2>&1 ; then export POLICY="manylinux_2_28" @@ -16,8 +17,9 @@ if command -v dnf >/dev/null 2>&1 ; then export AUDITWHEEL_POLICY=${POLICY} export AUDITWHEEL_ARCH=${REAL_ARCH} export AUDITWHEEL_PLAT=${POLICY}_${REAL_ARCH} + manylinux="manylinux_2_28" fi python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist +RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 83e24ab3ff1..8da756fea5e 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -3,8 +3,18 @@ set -eou pipefail +# Set the manylinux version to pull the newer ABI wheels when testing on Ubuntu 20.04 or later. +manylinux="manylinux_2_17" +if command -v lsb_release >/dev/null 2>&1 ; then + release_info=$(lsb_release -r) + major_version=$(echo "$release_info" | awk '{print $2}' | cut -d. -f1) + if [[ ${major_version} -ge 20 ]]; then + manylinux="manylinux_2_28" + fi +fi + RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/cudf*.whl)[test] From 4acfee8a0df9c6b53401f5125a8255af7a3b48cf Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 28 Oct 2023 13:51:25 -0700 Subject: [PATCH 09/20] Also use the new workflow for tests --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0feadc21be7..255cf77303c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -106,7 +106,7 @@ jobs: wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@feat/2_28_wheels with: build_type: pull-request script: ci/test_wheel_cudf.sh From 2578216310484424580d78ef546d86c7e8176450 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sun, 29 Oct 2023 16:28:21 -0700 Subject: [PATCH 10/20] Remove logic that has been upstreamed to the containers --- ci/build_wheel.sh | 3 --- ci/build_wheel_cudf.sh | 5 ----- ci/test_wheel_cudf.sh | 4 +++- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 02b47327833..06d0c3c7a56 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -6,9 +6,6 @@ set -euo pipefail package_name=$1 package_dir=$2 -if command -v dnf >/dev/null 2>&1 ; then - dnf install -y jq -fi source rapids-configure-sccache source rapids-date-string diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index eefd2a07de0..120bc48d3a5 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -12,11 +12,6 @@ export SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" manylinux="manylinux_2_17" mkdir -p ${package_dir}/final_dist if command -v dnf >/dev/null 2>&1 ; then - export POLICY="manylinux_2_28" - export REAL_ARCH=$(arch) - export AUDITWHEEL_POLICY=${POLICY} - export AUDITWHEEL_ARCH=${REAL_ARCH} - export AUDITWHEEL_PLAT=${POLICY}_${REAL_ARCH} manylinux="manylinux_2_28" fi python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 8da756fea5e..edb140ee868 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -3,7 +3,9 @@ set -eou pipefail -# Set the manylinux version to pull the newer ABI wheels when testing on Ubuntu 20.04 or later. +# Set the manylinux version to pull the newer ABI wheels when testing on Ubuntu +# 20.04 or later. Note that this logic is specific to the current set of +# citestwheel images and would need to change if the images change. manylinux="manylinux_2_17" if command -v lsb_release >/dev/null 2>&1 ; then release_info=$(lsb_release -r) From ea6f373bbc5b92809d3b7ad75d3b2b85b75fb911 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Oct 2023 10:32:20 -0700 Subject: [PATCH 11/20] Revert to main workflows branch --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 255cf77303c..2da47f42c29 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -98,7 +98,7 @@ jobs: wheel-build-cudf: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@feat/2_28_wheels + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 with: build_type: pull-request build-2_28-wheels: "true" @@ -106,7 +106,7 @@ jobs: wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@feat/2_28_wheels + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 with: build_type: pull-request script: ci/test_wheel_cudf.sh From f715b18b27622fbd81ae43c90521b1c699ef24d3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Oct 2023 10:35:37 -0700 Subject: [PATCH 12/20] Remove now unnecessary conditional --- cpp/cmake/thirdparty/get_arrow.cmake | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index ff34610a8e3..af2697475bd 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -424,22 +424,12 @@ function(find_and_configure_arrow VERSION BUILD_STATIC ENABLE_S3 ENABLE_ORC ENAB endfunction() if(NOT DEFINED CUDF_VERSION_Arrow) - # Temporarily use Arrow 12.0.1 in wheels and Arrow 13.0.0 otherwise - if(USE_LIBARROW_FROM_PYARROW) - set(CUDF_VERSION_Arrow - # This version must be kept in sync with the libarrow version pinned for builds in - # dependencies.yaml. - 13.0.0 - CACHE STRING "The version of Arrow to find (or build)" - ) - else() - set(CUDF_VERSION_Arrow - # This version must be kept in sync with the libarrow version pinned for builds in - # dependencies.yaml. - 13.0.0 - CACHE STRING "The version of Arrow to find (or build)" - ) - endif() + set(CUDF_VERSION_Arrow + # This version must be kept in sync with the libarrow version pinned for builds in + # dependencies.yaml. + 13.0.0 + CACHE STRING "The version of Arrow to find (or build)" + ) endif() find_and_configure_arrow( From a209ad14077da4944e0eb357a523e815e0fee757 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Oct 2023 11:25:56 -0700 Subject: [PATCH 13/20] Update the commands used to determine the manylinux version --- ci/build_wheel_cudf.sh | 7 ++----- ci/test_wheel_cudf.sh | 13 ++----------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 120bc48d3a5..0a616a76b7b 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -9,11 +9,8 @@ export SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" ./ci/build_wheel.sh cudf ${package_dir} -manylinux="manylinux_2_17" -mkdir -p ${package_dir}/final_dist -if command -v dnf >/dev/null 2>&1 ; then - manylinux="manylinux_2_28" -fi +# Set the manylinux version used for uploading the wheels +manylinux="manylinux_$(ldd --version | head -1 | grep -o "[0-9]\.[0-9]\+" | sed 's/\./_/g')" python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index edb140ee868..944ec86b15a 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -3,17 +3,8 @@ set -eou pipefail -# Set the manylinux version to pull the newer ABI wheels when testing on Ubuntu -# 20.04 or later. Note that this logic is specific to the current set of -# citestwheel images and would need to change if the images change. -manylinux="manylinux_2_17" -if command -v lsb_release >/dev/null 2>&1 ; then - release_info=$(lsb_release -r) - major_version=$(echo "$release_info" | awk '{print $2}' | cut -d. -f1) - if [[ ${major_version} -ge 20 ]]; then - manylinux="manylinux_2_28" - fi -fi +# Set the manylinux version used for downloading the wheels +manylinux="manylinux_$(ldd --version | head -1 | grep -o "[0-9]\.[0-9]\+" | sed 's/\./_/g')" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist From 1b4b580534b22e8999fddd59964c94093eddcc94 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Oct 2023 11:27:45 -0700 Subject: [PATCH 14/20] Update comment --- cpp/cmake/thirdparty/get_arrow.cmake | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index af2697475bd..ee17c3476e7 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -53,15 +53,15 @@ function(find_libarrow_in_python_wheel PYARROW_VERSION) find_package(Arrow ${PYARROW_VERSION} MODULE REQUIRED GLOBAL) add_library(arrow_shared ALIAS Arrow::Arrow) - # When using the libarrow inside a wheel we must build libcudf with the old ABI because pyarrow's - # `libarrow.so` is compiled for manylinux2014 (centos7 toolchain) which uses the old ABI. Note - # that these flags will often be redundant because we build wheels in manylinux containers that - # actually have the old libc++ anyway, but setting them explicitly ensures correct and consistent - # behavior in all other cases such as aarch builds on newer manylinux or testing builds in newer - # containers. Note that tests will not build successfully without also propagating these options - # to builds of GTest. Similarly, benchmarks will not work without updating GBench (and possibly - # NVBench) builds. We are currently ignoring these limitations since we don't anticipate using - # this feature except for building wheels. + # When using the libarrow inside a wheel, whether or not libcudf may be built + # using the new C++11 ABI is dependent on whether the libarrow inside the + # wheel was compiled using that ABI because we need the arrow library that we + # bundle in cudf to be ABI-compatible with the one inside pyarrow. Note that + # tests will not build successfully without also propagating these options to + # builds of GTest. Similarly, benchmarks will not work without updating + # GBench (and possibly NVBench) builds. We are currently ignoring these + # limitations since we don't anticipate using this feature except for + # building wheels. EXECUTE_PROCESS( COMMAND ${CMAKE_C_COMPILER} -print-file-name=libc.so.6 OUTPUT_VARIABLE GLIBC_EXECUTABLE From 6a41945c4e2ba9ae05342e76fe84e5bff29809d9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Oct 2023 11:30:14 -0700 Subject: [PATCH 15/20] Merge dependency lists --- cpp/cmake/thirdparty/get_arrow.cmake | 14 ++++++++------ dependencies.yaml | 13 ++----------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index ee17c3476e7..a86caa19bd0 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -56,12 +56,14 @@ function(find_libarrow_in_python_wheel PYARROW_VERSION) # When using the libarrow inside a wheel, whether or not libcudf may be built # using the new C++11 ABI is dependent on whether the libarrow inside the # wheel was compiled using that ABI because we need the arrow library that we - # bundle in cudf to be ABI-compatible with the one inside pyarrow. Note that - # tests will not build successfully without also propagating these options to - # builds of GTest. Similarly, benchmarks will not work without updating - # GBench (and possibly NVBench) builds. We are currently ignoring these - # limitations since we don't anticipate using this feature except for - # building wheels. + # bundle in cudf to be ABI-compatible with the one inside pyarrow. We + # determine what options to use by checking the glibc version on the current + # system, which is also how pip determines which manylinux-versioned pyarrow + # wheel to install. Note that tests will not build successfully without also + # propagating these options to builds of GTest. Similarly, benchmarks will + # not work without updating GBench (and possibly NVBench) builds. We are + # currently ignoring these limitations since we don't anticipate using this + # feature except for building wheels. EXECUTE_PROCESS( COMMAND ${CMAKE_C_COMPILER} -print-file-name=libc.so.6 OUTPUT_VARIABLE GLIBC_EXECUTABLE diff --git a/dependencies.yaml b/dependencies.yaml index 8e987c954b2..bb61e244b97 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -242,12 +242,7 @@ dependencies: - cython>=3.0.0 # TODO: Pin to numpy<1.25 until cudf requires pandas 2 - &numpy numpy>=1.21,<1.25 - - output_types: [conda] - packages: - # Hard pin the patch version used during the build. This must be kept - # in sync with the version pinned in get_arrow.cmake. - - pyarrow==13.0.0.* - - output_types: [requirements, pyproject] + - output_types: [conda, requirements, pyproject] packages: # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. @@ -272,11 +267,7 @@ dependencies: - libarrow==13.* pyarrow_run: common: - - output_types: [conda] - packages: - # Allow runtime version to float up to minor version - - pyarrow==13.* - - output_types: [requirements, pyproject] + - output_types: [conda, requirements, pyproject] packages: # Allow runtime version to float up to minor version - pyarrow==13.* From d1d8fa24bcbb732a6fc611e23045aaeabee636ff Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Oct 2023 11:32:46 -0700 Subject: [PATCH 16/20] Fix format --- cpp/cmake/thirdparty/get_arrow.cmake | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index a86caa19bd0..469b968eefd 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -53,17 +53,15 @@ function(find_libarrow_in_python_wheel PYARROW_VERSION) find_package(Arrow ${PYARROW_VERSION} MODULE REQUIRED GLOBAL) add_library(arrow_shared ALIAS Arrow::Arrow) - # When using the libarrow inside a wheel, whether or not libcudf may be built - # using the new C++11 ABI is dependent on whether the libarrow inside the - # wheel was compiled using that ABI because we need the arrow library that we - # bundle in cudf to be ABI-compatible with the one inside pyarrow. We - # determine what options to use by checking the glibc version on the current - # system, which is also how pip determines which manylinux-versioned pyarrow - # wheel to install. Note that tests will not build successfully without also - # propagating these options to builds of GTest. Similarly, benchmarks will - # not work without updating GBench (and possibly NVBench) builds. We are - # currently ignoring these limitations since we don't anticipate using this - # feature except for building wheels. + # When using the libarrow inside a wheel, whether or not libcudf may be built using the new C++11 + # ABI is dependent on whether the libarrow inside the wheel was compiled using that ABI because we + # need the arrow library that we bundle in cudf to be ABI-compatible with the one inside pyarrow. + # We determine what options to use by checking the glibc version on the current system, which is + # also how pip determines which manylinux-versioned pyarrow wheel to install. Note that tests will + # not build successfully without also propagating these options to builds of GTest. Similarly, + # benchmarks will not work without updating GBench (and possibly NVBench) builds. We are currently + # ignoring these limitations since we don't anticipate using this feature except for building + # wheels. EXECUTE_PROCESS( COMMAND ${CMAKE_C_COMPILER} -print-file-name=libc.so.6 OUTPUT_VARIABLE GLIBC_EXECUTABLE From dfecf8dc30e13e320f5680901c17388d736e3912 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Oct 2023 12:48:46 -0700 Subject: [PATCH 17/20] Simplify the build logic and update the test logic --- ci/build_wheel_cudf.sh | 4 +--- ci/test_wheel_cudf.sh | 10 ++++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 0a616a76b7b..1586fec3a22 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -9,9 +9,7 @@ export SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" ./ci/build_wheel.sh cudf ${package_dir} -# Set the manylinux version used for uploading the wheels -manylinux="manylinux_$(ldd --version | head -1 | grep -o "[0-9]\.[0-9]\+" | sed 's/\./_/g')" python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist +RAPIDS_PY_WHEEL_NAME="cudf_${AUDITWHEEL_POLICY}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 944ec86b15a..dea81bfb3ea 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -3,8 +3,14 @@ set -eou pipefail -# Set the manylinux version used for downloading the wheels -manylinux="manylinux_$(ldd --version | head -1 | grep -o "[0-9]\.[0-9]\+" | sed 's/\./_/g')" +# Set the manylinux version used for downloading the wheels so that we test the +# newer ABI wheels on the newer images that support their installation. +glibc_minor_version=$(ldd --version | head -1 | grep -o "[0-9]\.[0-9]\+" | tail -1 | cut -d '.' -f2) +manylinux_version="2_17" +if [[ ${glibc_minor_version} -ge 28 ]]; then + manylinux_version="2_28" +fi +manylinux="manylinux_${manylinux_version}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist From 60162c7d0dfeaef7022e7c7ccbbf73915fa4a2d2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Oct 2023 18:16:09 -0700 Subject: [PATCH 18/20] Ignore SIGPIPE error --- ci/test_wheel_cudf.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index dea81bfb3ea..8c42651e299 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -5,7 +5,11 @@ set -eou pipefail # Set the manylinux version used for downloading the wheels so that we test the # newer ABI wheels on the newer images that support their installation. +# Need to disable pipefail for the head not to fail, see +# https://stackoverflow.com/questions/19120263/why-exit-code-141-with-grep-q +set +o pipefail glibc_minor_version=$(ldd --version | head -1 | grep -o "[0-9]\.[0-9]\+" | tail -1 | cut -d '.' -f2) +set -o pipefail manylinux_version="2_17" if [[ ${glibc_minor_version} -ge 28 ]]; then manylinux_version="2_28" From 909a5295fdb2707c0bf31956169d904762c4f2e3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Oct 2023 19:17:56 -0700 Subject: [PATCH 19/20] Also get the right cudf wheel when running dask-cudf --- ci/test_wheel_dask_cudf.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index f89aa43c20a..234f5fdd859 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -7,7 +7,20 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist # Download the cudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep +# Set the manylinux version used for downloading the wheels so that we test the +# newer ABI wheels on the newer images that support their installation. +# Need to disable pipefail for the head not to fail, see +# https://stackoverflow.com/questions/19120263/why-exit-code-141-with-grep-q +set +o pipefail +glibc_minor_version=$(ldd --version | head -1 | grep -o "[0-9]\.[0-9]\+" | tail -1 | cut -d '.' -f2) +set -o pipefail +manylinux_version="2_17" +if [[ ${glibc_minor_version} -ge 28 ]]; then + manylinux_version="2_28" +fi +manylinux="manylinux_${manylinux_version}" + +RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist python -m pip install --no-deps ./local-cudf-dep/cudf*.whl # Always install latest dask for testing From 9e5589f4789600f5c1bfc3c05b075b4053c20cf2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Oct 2023 20:58:50 -0700 Subject: [PATCH 20/20] Use the right directory --- ci/test_wheel_dask_cudf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 234f5fdd859..118bea753d0 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -20,7 +20,7 @@ if [[ ${glibc_minor_version} -ge 28 ]]; then fi manylinux="manylinux_${manylinux_version}" -RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep python -m pip install --no-deps ./local-cudf-dep/cudf*.whl # Always install latest dask for testing