From 3e8be1c3e53664f2d8647ace2696099d88115762 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 17 Oct 2023 19:43:33 -0700 Subject: [PATCH 01/16] Temporarily add hard pins of a specific arrow build --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 2 +- dependencies.yaml | 6 ++++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8976a4b14cb..096dcd5e2b6 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -40,7 +40,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow==12.0.1.* +- libarrow==12.0.1.*=hb6645e0_10_cpu - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index f54d78593c3..42da06e4bb5 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -41,7 +41,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow==12.0.1.* +- libarrow==12.0.1.*=hb6645e0_10_cpu - libcufile-dev - libcurand-dev - libkvikio==23.12.* diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index b1f5b083e06..956c8c47fd2 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -26,7 +26,7 @@ aws_sdk_cpp_version: - "<1.11" libarrow_version: - - "=12" + - "==12.0.1.*=hb6645e0_10_cpu" dlpack_version: - ">=0.5,<0.6.0a0" diff --git a/dependencies.yaml b/dependencies.yaml index c19e8765be3..a0359671f87 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -225,7 +225,9 @@ dependencies: - &gmock gmock>=1.13.0 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - libarrow==12.0.1.* + # Temporarily building a specific build string in order to choose one + # that is compatible with the aws-sdk-cpp version required above + - libarrow==12.0.1.*=hb6645e0_10_cpu - librdkafka>=1.9.0,<1.10.0a0 # Align nvcomp version with rapids-cmake - nvcomp==2.6.1 @@ -263,7 +265,7 @@ dependencies: - output_types: [conda, requirements] packages: # Allow runtime version to float up to minor version - - libarrow==12.* + - libarrow==12.0.1.*=hb6645e0_10_cpu pyarrow_run: common: - output_types: [conda, requirements, pyproject] From 985313f404d67a393d275efedd85ea08fe759f41 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 09:06:21 -0700 Subject: [PATCH 02/16] Fix pinnings --- conda/recipes/libcudf/conda_build_config.yaml | 5 +++- conda/recipes/libcudf/meta.yaml | 3 +- dependencies.yaml | 28 +++++++++++++++---- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 956c8c47fd2..9785edcbdfd 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -25,9 +25,12 @@ gtest_version: aws_sdk_cpp_version: - "<1.11" -libarrow_version: +libarrow_x86_version: - "==12.0.1.*=hb6645e0_10_cpu" +libarrow_aarch64_version: + - "==12.0.1.*=hf79fbb4_10_cpu" + dlpack_version: - ">=0.5,<0.6.0a0" diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index 28357f0d96d..4e3a06ef154 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -65,7 +65,8 @@ requirements: {% endif %} - cuda-version ={{ cuda_version }} - nvcomp {{ nvcomp_version }} - - libarrow {{ libarrow_version }} + - libarrow {{ libarrow_x86_version }} # [linux64] + - libarrow {{ libarrow_aarch64_version }} # [aarch64] - dlpack {{ dlpack_version }} - librdkafka {{ librdkafka_version }} - fmt {{ fmt_version }} diff --git a/dependencies.yaml b/dependencies.yaml index a0359671f87..d9220e5067b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -227,11 +227,21 @@ dependencies: # in sync with the version pinned in get_arrow.cmake. # Temporarily building a specific build string in order to choose one # that is compatible with the aws-sdk-cpp version required above - - libarrow==12.0.1.*=hb6645e0_10_cpu - librdkafka>=1.9.0,<1.10.0a0 # Align nvcomp version with rapids-cmake - nvcomp==2.6.1 - spdlog>=1.11.0,<1.12 + specific: + - output_types: conda + matrices: + - matrix: + arch: x86_64 + packages: + - libarrow==12.0.1.*=hb6645e0_10_cpu + - matrix: + arch: aarch64 + packages: + - libarrow==12.0.1.*=hf79fbb4_10_cpu build_wheels: common: - output_types: pyproject @@ -261,11 +271,17 @@ dependencies: packages: - protoc-wheel libarrow_run: - common: - - output_types: [conda, requirements] - packages: - # Allow runtime version to float up to minor version - - libarrow==12.0.1.*=hb6645e0_10_cpu + specific: + - output_types: conda + matrices: + - matrix: + arch: x86_64 + packages: + - libarrow==12.0.1.*=hb6645e0_10_cpu + - matrix: + arch: aarch64 + packages: + - libarrow==12.0.1.*=hf79fbb4_10_cpu pyarrow_run: common: - output_types: [conda, requirements, pyproject] From 2d1b406d228a24c6f9282821e3a927ec8fe06cce Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 10:33:54 -0700 Subject: [PATCH 03/16] Fix recipe --- conda/recipes/libcudf/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index 4e3a06ef154..39370c58b61 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -105,7 +105,8 @@ outputs: - nvcomp {{ nvcomp_version }} - librmm ={{ minor_version }} - libkvikio ={{ minor_version }} - - libarrow {{ libarrow_version }} + - libarrow {{ libarrow_x86_version }} # [linux64] + - libarrow {{ libarrow_aarch64_version }} # [aarch64] - dlpack {{ dlpack_version }} - gtest {{ gtest_version }} - gmock {{ gtest_version }} From ff8a5b1b3b10a8e72f69dd96adf389c6b7538f77 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 11:44:05 -0700 Subject: [PATCH 04/16] Fix build version --- conda/recipes/libcudf/conda_build_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 9785edcbdfd..570ed11fc5f 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -26,10 +26,10 @@ aws_sdk_cpp_version: - "<1.11" libarrow_x86_version: - - "==12.0.1.*=hb6645e0_10_cpu" + - "==12.0.1=hb6645e0_10_cpu" libarrow_aarch64_version: - - "==12.0.1.*=hf79fbb4_10_cpu" + - "==12.0.1=hf79fbb4_10_cpu" dlpack_version: - ">=0.5,<0.6.0a0" From f98c903b23bcc20305d12d8269e8a9964747fdcd Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 11:57:42 -0700 Subject: [PATCH 05/16] Also fix dependencies.yaml --- dependencies.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index d9220e5067b..e0b5ab23d68 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -237,11 +237,11 @@ dependencies: - matrix: arch: x86_64 packages: - - libarrow==12.0.1.*=hb6645e0_10_cpu + - libarrow==12.0.1=hb6645e0_10_cpu - matrix: arch: aarch64 packages: - - libarrow==12.0.1.*=hf79fbb4_10_cpu + - libarrow==12.0.1=hf79fbb4_10_cpu build_wheels: common: - output_types: pyproject @@ -277,11 +277,11 @@ dependencies: - matrix: arch: x86_64 packages: - - libarrow==12.0.1.*=hb6645e0_10_cpu + - libarrow==12.0.1=hb6645e0_10_cpu - matrix: arch: aarch64 packages: - - libarrow==12.0.1.*=hf79fbb4_10_cpu + - libarrow==12.0.1=hf79fbb4_10_cpu pyarrow_run: common: - output_types: [conda, requirements, pyproject] From 116c82c5c582967385911640b50a594905821451 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 12:27:00 -0700 Subject: [PATCH 06/16] Run dfg --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-120_arch-x86_64.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 096dcd5e2b6..2ca30a7b9ab 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -40,7 +40,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow==12.0.1.*=hb6645e0_10_cpu +- libarrow==12.0.1=hb6645e0_10_cpu - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 42da06e4bb5..8dbbe8acc6b 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -41,7 +41,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow==12.0.1.*=hb6645e0_10_cpu +- libarrow==12.0.1=hb6645e0_10_cpu - libcufile-dev - libcurand-dev - libkvikio==23.12.* From c4e1a82c4a5a8c1c07f5c41585302b17e4f8ee65 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 14:13:06 -0700 Subject: [PATCH 07/16] Specify libarrow dependency in the docs env as well --- dependencies.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dependencies.yaml b/dependencies.yaml index e0b5ab23d68..7de2c828fa0 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -62,6 +62,7 @@ files: includes: - cudatoolkit - docs + - libarrow_run - py_version py_build_cudf: output: pyproject From 415bfbef2b386c1075d21264bdfcbdeb5b0bf994 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 16:05:10 -0700 Subject: [PATCH 08/16] Add cuda-gdb to CUDA 12 requirements for cudf-kafka --- conda/environments/all_cuda-120_arch-x86_64.yaml | 1 + conda/recipes/cudf_kafka/meta.yaml | 3 +++ dependencies.yaml | 1 + 3 files changed, 5 insertions(+) diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 8dbbe8acc6b..c3a41e78680 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -17,6 +17,7 @@ dependencies: - cachetools - cmake>=3.26.4 - cuda-cudart-dev +- cuda-gdb - cuda-nvcc - cuda-nvrtc-dev - cuda-nvtx-dev diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml index ec0cc402511..4c9d4441b62 100644 --- a/conda/recipes/cudf_kafka/meta.yaml +++ b/conda/recipes/cudf_kafka/meta.yaml @@ -41,6 +41,9 @@ requirements: - {{ compiler('cxx') }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} + {% if cuda_major == "12" %} + - cuda-gdb + {% endif %} host: - python - cython >=3.0.0 diff --git a/dependencies.yaml b/dependencies.yaml index 7de2c828fa0..616347b3f6f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -301,6 +301,7 @@ dependencies: - cuda-nvrtc-dev - cuda-nvtx-dev - libcurand-dev + - cuda-gdb - matrix: cuda: "11.8" packages: From fb5e9217931b06a262bb8366aac3b2c0a4ddaf19 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 18:13:21 -0700 Subject: [PATCH 09/16] Make sure cuda.h is installed and patch path during conda build --- conda/recipes/cudf_kafka/build.sh | 9 +++++++++ conda/recipes/cudf_kafka/meta.yaml | 3 +++ 2 files changed, 12 insertions(+) diff --git a/conda/recipes/cudf_kafka/build.sh b/conda/recipes/cudf_kafka/build.sh index 5d8720f1c98..2fe9a6a4556 100644 --- a/conda/recipes/cudf_kafka/build.sh +++ b/conda/recipes/cudf_kafka/build.sh @@ -1,4 +1,13 @@ # Copyright (c) 2020-2022, NVIDIA CORPORATION. # This assumes the script is executed from the root of the repo directory +# Need to set CUDA_HOME inside conda environments because the hacked together +# setup.py for cudf-kafka searches that way. +# TODO: Remove after https://github.com/rapidsai/cudf/pull/14292 updates +# cudf_kafka to use scikit-build +target_name="x86_64-linux" +if [[ ! $(arch) == "x86_64" ]]; then + target_name="sbsa-linux" +fi +export CUDA_HOME="${PREFIX}/targets/${target_name}/" ./build.sh -v cudf_kafka diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml index 4c9d4441b62..3fc2c492f10 100644 --- a/conda/recipes/cudf_kafka/meta.yaml +++ b/conda/recipes/cudf_kafka/meta.yaml @@ -51,6 +51,9 @@ requirements: - cudf ={{ version }} - libcudf_kafka ={{ version }} - setuptools + {% if cuda_major == "12" %} + - cuda-cudart-dev + {% endif %} run: - python - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} From 240362b115210978688ac12d4c388e715a822749 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 18:14:14 -0700 Subject: [PATCH 10/16] Add some comments and clean up --- conda/recipes/cudf_kafka/meta.yaml | 2 ++ dependencies.yaml | 16 ++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml index 3fc2c492f10..f1c3e19fe91 100644 --- a/conda/recipes/cudf_kafka/meta.yaml +++ b/conda/recipes/cudf_kafka/meta.yaml @@ -41,6 +41,8 @@ requirements: - {{ compiler('cxx') }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} + # TODO: Remove after https://github.com/rapidsai/cudf/pull/14292 updates + # cudf_kafka to use scikit-build {% if cuda_major == "12" %} - cuda-gdb {% endif %} diff --git a/dependencies.yaml b/dependencies.yaml index 616347b3f6f..2d96b86e746 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -226,8 +226,6 @@ dependencies: - &gmock gmock>=1.13.0 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - # Temporarily building a specific build string in order to choose one - # that is compatible with the aws-sdk-cpp version required above - librdkafka>=1.9.0,<1.10.0a0 # Align nvcomp version with rapids-cmake - nvcomp==2.6.1 @@ -235,14 +233,18 @@ dependencies: specific: - output_types: conda matrices: + # Temporarily pinning a specific build string for libarrow in order + # to choose one that is compatible with the aws-sdk-cpp version + # we require due to https://github.com/aws/aws-sdk-cpp/issues/2681 + # (see also https://github.com/rapidsai/cudf/pull/14173). - matrix: arch: x86_64 packages: - - libarrow==12.0.1=hb6645e0_10_cpu + - &libarrow_x86 libarrow==12.0.1=hb6645e0_10_cpu - matrix: arch: aarch64 packages: - - libarrow==12.0.1=hf79fbb4_10_cpu + - &libarrow_aarch libarrow==12.0.1=hf79fbb4_10_cpu build_wheels: common: - output_types: pyproject @@ -278,11 +280,11 @@ dependencies: - matrix: arch: x86_64 packages: - - libarrow==12.0.1=hb6645e0_10_cpu + - *libarrow_x86 - matrix: arch: aarch64 packages: - - libarrow==12.0.1=hf79fbb4_10_cpu + - *libarrow_aarch pyarrow_run: common: - output_types: [conda, requirements, pyproject] @@ -301,6 +303,8 @@ dependencies: - cuda-nvrtc-dev - cuda-nvtx-dev - libcurand-dev + # TODO: Remove after https://github.com/rapidsai/cudf/pull/14292 updates + # cudf_kafka to use scikit-build - cuda-gdb - matrix: cuda: "11.8" From 05ece0a8d2b14dc222cc24842416af2b8258a94e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 18:40:12 -0700 Subject: [PATCH 11/16] Fix copyright --- conda/recipes/cudf_kafka/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/cudf_kafka/build.sh b/conda/recipes/cudf_kafka/build.sh index 2fe9a6a4556..925cc4d936f 100644 --- a/conda/recipes/cudf_kafka/build.sh +++ b/conda/recipes/cudf_kafka/build.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. # This assumes the script is executed from the root of the repo directory # Need to set CUDA_HOME inside conda environments because the hacked together From f5a4e63f6e1b723126476c977c6793a596ec89c1 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 19:47:34 -0700 Subject: [PATCH 12/16] Only set CUDA_HOME for CUDA 12 builds --- conda/recipes/cudf_kafka/build.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/conda/recipes/cudf_kafka/build.sh b/conda/recipes/cudf_kafka/build.sh index 925cc4d936f..7ed0f2af9ca 100644 --- a/conda/recipes/cudf_kafka/build.sh +++ b/conda/recipes/cudf_kafka/build.sh @@ -5,9 +5,13 @@ # setup.py for cudf-kafka searches that way. # TODO: Remove after https://github.com/rapidsai/cudf/pull/14292 updates # cudf_kafka to use scikit-build -target_name="x86_64-linux" -if [[ ! $(arch) == "x86_64" ]]; then - target_name="sbsa-linux" +CUDA_MAJOR=${RAPIDS_CUDA_VERSION%%.*} +if [[ ${CUDA_MAJOR} == "12" ]]; then + target_name="x86_64-linux" + if [[ ! $(arch) == "x86_64" ]]; then + target_name="sbsa-linux" + fi + export CUDA_HOME="${PREFIX}/targets/${target_name}/" + echo ${CUDA_HOME} fi -export CUDA_HOME="${PREFIX}/targets/${target_name}/" -./build.sh -v cudf_kafka +#./build.sh -v cudf_kafka From 0b5bab45bfaab47025e61e61ff65b5ca1da293ae Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 18 Oct 2023 22:14:43 -0700 Subject: [PATCH 13/16] Put back build command... --- conda/recipes/cudf_kafka/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/cudf_kafka/build.sh b/conda/recipes/cudf_kafka/build.sh index 7ed0f2af9ca..306aa904d00 100644 --- a/conda/recipes/cudf_kafka/build.sh +++ b/conda/recipes/cudf_kafka/build.sh @@ -14,4 +14,4 @@ if [[ ${CUDA_MAJOR} == "12" ]]; then export CUDA_HOME="${PREFIX}/targets/${target_name}/" echo ${CUDA_HOME} fi -#./build.sh -v cudf_kafka +./build.sh -v cudf_kafka From 68818b23d45d55a6622efce5b4a74574317ba08c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 19 Oct 2023 10:08:50 -0700 Subject: [PATCH 14/16] Make sure RAPIDS_CUDA_VERSION is available in the build environment --- conda/recipes/cudf_kafka/build.sh | 1 - conda/recipes/cudf_kafka/meta.yaml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/cudf_kafka/build.sh b/conda/recipes/cudf_kafka/build.sh index 306aa904d00..f4bb6e1bc91 100644 --- a/conda/recipes/cudf_kafka/build.sh +++ b/conda/recipes/cudf_kafka/build.sh @@ -12,6 +12,5 @@ if [[ ${CUDA_MAJOR} == "12" ]]; then target_name="sbsa-linux" fi export CUDA_HOME="${PREFIX}/targets/${target_name}/" - echo ${CUDA_HOME} fi ./build.sh -v cudf_kafka diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml index f1c3e19fe91..a0edbe5bf46 100644 --- a/conda/recipes/cudf_kafka/meta.yaml +++ b/conda/recipes/cudf_kafka/meta.yaml @@ -33,6 +33,7 @@ build: - SCCACHE_S3_KEY_PREFIX=cudf-kafka-linux64 # [linux64] - SCCACHE_S3_USE_SSL - SCCACHE_S3_NO_CREDENTIALS + - RAPIDS_CUDA_VERSION requirements: build: From ff6fe4a7e447fc458f9b9d2bf445ee761631717a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 19 Oct 2023 10:10:03 -0700 Subject: [PATCH 15/16] Downgrade to arrow 12.0.0 --- .../all_cuda-118_arch-x86_64.yaml | 4 +-- .../all_cuda-120_arch-x86_64.yaml | 4 +-- conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 7 ++-- conda/recipes/libcudf/meta.yaml | 6 ++-- cpp/cmake/thirdparty/get_arrow.cmake | 2 +- dependencies.yaml | 34 ++++--------------- python/cudf/pyproject.toml | 4 +-- python/cudf_kafka/pyproject.toml | 2 +- 9 files changed, 19 insertions(+), 46 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 2ca30a7b9ab..2c79cbb6b6c 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -40,7 +40,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow==12.0.1=hb6645e0_10_cpu +- libarrow==12.0.0.* - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 @@ -69,7 +69,7 @@ dependencies: - pre-commit - protobuf>=4.21,<5 - ptxcompiler -- pyarrow==12.0.1.* +- pyarrow==12.0.0.* - pydata-sphinx-theme - pyorc - pytest diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index c3a41e78680..c96b7428882 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -42,7 +42,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow==12.0.1=hb6645e0_10_cpu +- libarrow==12.0.0.* - libcufile-dev - libcurand-dev - libkvikio==23.12.* @@ -67,7 +67,7 @@ dependencies: - pip - pre-commit - protobuf>=4.21,<5 -- pyarrow==12.0.1.* +- pyarrow==12.0.0.* - pydata-sphinx-theme - pyorc - pytest diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 54b687faa69..16b064a262e 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -61,7 +61,7 @@ requirements: - scikit-build >=0.13.1 - setuptools - dlpack >=0.5,<0.6.0a0 - - pyarrow =12 + - pyarrow =12.0.0 - libcudf ={{ version }} - rmm ={{ minor_version }} {% if cuda_major == "11" %} diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 570ed11fc5f..4d33bb89220 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -25,11 +25,8 @@ gtest_version: aws_sdk_cpp_version: - "<1.11" -libarrow_x86_version: - - "==12.0.1=hb6645e0_10_cpu" - -libarrow_aarch64_version: - - "==12.0.1=hf79fbb4_10_cpu" +libarrow: + - "==12.0.0" dlpack_version: - ">=0.5,<0.6.0a0" diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index 39370c58b61..b9aff2a9c82 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -65,8 +65,7 @@ requirements: {% endif %} - cuda-version ={{ cuda_version }} - nvcomp {{ nvcomp_version }} - - libarrow {{ libarrow_x86_version }} # [linux64] - - libarrow {{ libarrow_aarch64_version }} # [aarch64] + - libarrow {{ libarrow }} - dlpack {{ dlpack_version }} - librdkafka {{ librdkafka_version }} - fmt {{ fmt_version }} @@ -105,8 +104,7 @@ outputs: - nvcomp {{ nvcomp_version }} - librmm ={{ minor_version }} - libkvikio ={{ minor_version }} - - libarrow {{ libarrow_x86_version }} # [linux64] - - libarrow {{ libarrow_aarch64_version }} # [aarch64] + - libarrow {{ libarrow }} - dlpack {{ dlpack_version }} - gtest {{ gtest_version }} - gmock {{ gtest_version }} diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index 10d3145a36f..c2d5cfbaf78 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -411,7 +411,7 @@ if(NOT DEFINED CUDF_VERSION_Arrow) set(CUDF_VERSION_Arrow # This version must be kept in sync with the libarrow version pinned for builds in # dependencies.yaml. - 12.0.1 + 12.0.0 CACHE STRING "The version of Arrow to find (or build)" ) endif() diff --git a/dependencies.yaml b/dependencies.yaml index 2d96b86e746..e8114fa5615 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -226,25 +226,11 @@ dependencies: - &gmock gmock>=1.13.0 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. + - &libarrow libarrow==12.0.0.* - librdkafka>=1.9.0,<1.10.0a0 # Align nvcomp version with rapids-cmake - nvcomp==2.6.1 - spdlog>=1.11.0,<1.12 - specific: - - output_types: conda - matrices: - # Temporarily pinning a specific build string for libarrow in order - # to choose one that is compatible with the aws-sdk-cpp version - # we require due to https://github.com/aws/aws-sdk-cpp/issues/2681 - # (see also https://github.com/rapidsai/cudf/pull/14173). - - matrix: - arch: x86_64 - packages: - - &libarrow_x86 libarrow==12.0.1=hb6645e0_10_cpu - - matrix: - arch: aarch64 - packages: - - &libarrow_aarch libarrow==12.0.1=hf79fbb4_10_cpu build_wheels: common: - output_types: pyproject @@ -258,7 +244,7 @@ dependencies: - cython>=3.0.0 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - pyarrow==12.0.1.* + - &pyarrow pyarrow==12.0.0.* # TODO: Pin to numpy<1.25 until cudf requires pandas 2 - &numpy numpy>=1.21,<1.25 build_python: @@ -274,23 +260,15 @@ dependencies: packages: - protoc-wheel libarrow_run: - specific: + common: - output_types: conda - matrices: - - matrix: - arch: x86_64 - packages: - - *libarrow_x86 - - matrix: - arch: aarch64 - packages: - - *libarrow_aarch + packages: + - *libarrow pyarrow_run: common: - output_types: [conda, requirements, pyproject] packages: - # Allow runtime version to float up to minor version - - pyarrow==12.* + - *pyarrow cudatoolkit: specific: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 39a8dca0267..ccb5d5d4416 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -8,7 +8,7 @@ requires = [ "ninja", "numpy>=1.21,<1.25", "protoc-wheel", - "pyarrow==12.0.1.*", + "pyarrow==12.0.0.*", "rmm==23.12.*", "scikit-build>=0.13.1", "setuptools", @@ -38,7 +38,7 @@ dependencies = [ "pandas>=1.3,<1.6.0dev0", "protobuf>=4.21,<5", "ptxcompiler", - "pyarrow==12.*", + "pyarrow==12.0.0.*", "rmm==23.12.*", "typing_extensions>=4.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 78a7a83ac3a..ff475e5a72e 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "cython>=3.0.0", "numpy>=1.21,<1.25", - "pyarrow==12.0.1.*", + "pyarrow==12.0.0.*", "setuptools", "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 0ffc3e7037f5275abb0f5e7fbb68c87999bf6259 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 19 Oct 2023 10:14:35 -0700 Subject: [PATCH 16/16] Add one more comment --- conda/recipes/cudf_kafka/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml index a0edbe5bf46..a79c23b7d98 100644 --- a/conda/recipes/cudf_kafka/meta.yaml +++ b/conda/recipes/cudf_kafka/meta.yaml @@ -33,6 +33,8 @@ build: - SCCACHE_S3_KEY_PREFIX=cudf-kafka-linux64 # [linux64] - SCCACHE_S3_USE_SSL - SCCACHE_S3_NO_CREDENTIALS + # TODO: Remove after https://github.com/rapidsai/cudf/pull/14292 updates + # cudf_kafka to use scikit-build - RAPIDS_CUDA_VERSION requirements: