Skip to content

Commit

Permalink
Consolidate C++ conda recipes and add libraft-tests package
Browse files Browse the repository at this point in the history
Signed-off-by: Jordan Jacobelli <jjacobelli@nvidia.com>
  • Loading branch information
jjacobelli committed May 9, 2022
1 parent 7a01581 commit 8ad57f5
Show file tree
Hide file tree
Showing 15 changed files with 186 additions and 304 deletions.
24 changes: 6 additions & 18 deletions ci/cpu/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -85,29 +85,17 @@ gpuci_mamba_retry install -c conda-forge boa
###############################################################################

if [ "$BUILD_LIBRAFT" == "1" ]; then
gpuci_logger "Building conda packages for libraft-nn, libraft-distance, and libraft-headers"
gpuci_logger "Building conda packages for libraft-nn, libraft-distance, libraft-headers and libraft-tests"
if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers
gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn
gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance
gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft
else
gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers
gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft
gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
mkdir -p ${CONDA_BLD_DIR}/libraft_headers/work
mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_headers/work

gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_nn
gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
mkdir -p ${CONDA_BLD_DIR}/libraft_nn/work
mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_nn/work

gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance
gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
mkdir -p ${CONDA_BLD_DIR}/libraft_distance/work
mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_distance/work
mkdir -p ${CONDA_BLD_DIR}/libraft/work
mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work
fi
else
gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance and libraft-headers"
gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance, libraft-headers and libraft-tests"
fi

if [ "$BUILD_RAFT" == '1' ]; then
Expand Down
25 changes: 5 additions & 20 deletions ci/cpu/upload.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,6 @@ fi

gpuci_logger "Get conda file output locations"

export LIBRAFT_HEADERS_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_headers --output`
export LIBRAFT_NN_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_nn --output`
export LIBRAFT_DISTANCE_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_distance --output`
export PYRAFT_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pyraft --python=$PYTHON --output`
export PYLIBRAFT_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pylibraft --python=$PYTHON --output`

################################################################################
# UPLOAD - Conda packages
Expand All @@ -44,24 +39,14 @@ export PYLIBRAFT_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${C
gpuci_logger "Starting conda uploads"

if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then

test -e ${LIBRAFT_HEADERS_FILE}
echo "Upload libraft-headers"
echo ${LIBRAFT_HEADERS_FILE}
gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress

test -e ${LIBRAFT_NN_FILE}
echo "Upload libraft-nn"
echo ${LIBRAFT_NN_FILE}
gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_NN_FILE} --no-progress

test -e ${LIBRAFT_DISTANCE_FILE}
echo "Upload libraft-distance"
echo ${LIBRAFT_DISTANCE_FILE}
gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_DISTANCE_FILE} --no-progress
LIBRAFT_FILES=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft --output)
echo "Upload libraft-headers, libraft-nn, libraft-distance and libraft-tests"
gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing --no-progress ${LIBRAFT_FILES}
fi

if [[ "$BUILD_RAFT" == "1" ]]; then
PYRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pyraft --python=$PYTHON --output)
PYLIBRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pylibraft --python=$PYTHON --output)
test -e ${PYRAFT_FILE}
echo "Upload pyraft"
echo ${PYRAFT_FILE}
Expand Down
68 changes: 14 additions & 54 deletions ci/gpu/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,11 @@ export HOME="$WORKSPACE"
cd "$WORKSPACE"
export GIT_DESCRIBE_TAG=`git describe --tags`
export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
unset GIT_DESCRIBE_TAG

# ucx-py version
export UCX_PY_VERSION='0.26.*'

export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"
export CMAKE_CXX_COMPILER_LAUNCHER="sccache"
export CMAKE_C_COMPILER_LAUNCHER="sccache"
export SCCACHE_S3_KEY_PREFIX="libraft-$(uname -m)"
export SCCACHE_BUCKET="rapids-sccache"
export SCCACHE_REGION="us-west-2"
export SCCACHE_IDLE_TIMEOUT="32768"

################################################################################
# SETUP - Check environment
################################################################################
Expand All @@ -47,40 +40,14 @@ env
gpuci_logger "Check GPU usage"
nvidia-smi

# temporary usage of gpuci_mamba_retry install with packages listed here, looking into
# using the repos yaml files for this
gpuci_logger "Activate conda env"
. /opt/conda/etc/profile.d/conda.sh
conda activate rapids
gpuci_logger "Installing packages needed for RAFT"
gpuci_mamba_retry install -y -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \
"cudatoolkit=${CUDA_REL}" \
"libcusolver>=11.2.1" \
"cudf=${MINOR_VERSION}" \
"rmm=${MINOR_VERSION}" \
"breathe" \
"dask-cudf=${MINOR_VERSION}" \
"dask-cuda=${MINOR_VERSION}" \
"ucx-py=${UCX_PY_VERSION}" \
"rapids-build-env=${MINOR_VERSION}.*" \
"rapids-notebook-env=${MINOR_VERSION}.*"

if [ "$(arch)" = "x86_64" ]; then
gpuci_mamba_retry install -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \
"rapids-doc-env=${MINOR_VERSION}.*"
fi

# Install the master version of dask, distributed, and dask-ml
gpuci_logger "Install the master version of dask and distributed"
set -x
pip install "git+https://github.com/dask/distributed.git@main" --upgrade --no-deps
pip install "git+https://github.com/dask/dask.git@main" --upgrade --no-deps
set +x

# Install pre-built conda packages from previous CI step
gpuci_logger "Install libraft conda packages from CPU job"
export LIBRAFT_CONDA_PACKAGES="$WORKSPACE/ci/artifacts/raft/cpu/.conda-bld/" # notice there is no `linux-64` here
gpuci_mamba_retry install -c "${LIBRAFT_CONDA_PACKAGES}" libraft-headers libraft-distance libraft-nn
CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/raft/cpu/.conda-bld/" # notice there is no `linux-64` here
gpuci_mamba_retry install -c "${CONDA_ARTIFACT_PATH}" libraft-headers libraft-distance libraft-nn libraft-tests

gpuci_logger "Check compiler versions"
python --version
Expand All @@ -96,24 +63,16 @@ conda list --show-channel-urls
# BUILD - Build RAFT tests
################################################################################

gpuci_logger "Adding ${CONDA_PREFIX}/lib to LD_LIBRARY_PATH"

export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH

gpuci_logger "Build C++ and Python targets"
# These should link against the existing shared libs
if hasArg --skip-tests; then
"$WORKSPACE/build.sh" pyraft pylibraft libraft -v
else
"$WORKSPACE/build.sh" pyraft pylibraft libraft tests bench -v
fi
gpuci_logger "Build and install Python targets"
CONDA_BLD_DIR="$WORKSPACE/.conda-bld"
gpuci_mamba_retry install boa
gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/pyraft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}"
gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/pylibraft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}"
gpuci_mamba_retry install -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" pyraft pylibraft

gpuci_logger "sccache stats"
sccache --show-stats

gpuci_logger "Building docs"
"$WORKSPACE/build.sh" docs -v

################################################################################
# TEST - Run GoogleTest and py.tests for RAFT
################################################################################
Expand All @@ -127,18 +86,19 @@ gpuci_logger "Check GPU usage"
nvidia-smi

gpuci_logger "GoogleTest for raft"
cd "$WORKSPACE/cpp/build"
GTEST_OUTPUT="xml:$WORKSPACE/test-results/raft_cpp/" ./test_raft
cd "$WORKSPACE"
GTEST_OUTPUT="xml:$WORKSPACE/test-results/raft_cpp/" "$CONDA_PREFIX/bin/libraft/gtests/test_raft"

gpuci_logger "Python pytest for pyraft"
cd "$WORKSPACE/python/raft"
cd "$WORKSPACE/python/raft/raft/test"
python -m pytest --cache-clear --junitxml="$WORKSPACE/junit-pyraft.xml" -v -s

gpuci_logger "Python pytest for pylibraft"
cd "$WORKSPACE/python/pylibraft"
cd "$WORKSPACE/python/pylibraft/pylibraft/test"
python -m pytest --cache-clear --junitxml="$WORKSPACE/junit-pylibraft.xml" -v -s

if [ "$(arch)" = "x86_64" ]; then
gpuci_logger "Building docs"
gpuci_mamba_retry install "rapids-doc-env=${MINOR_VERSION}.*"
"$WORKSPACE/build.sh" docs -v
fi
4 changes: 4 additions & 0 deletions conda/recipes/libraft/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Copyright (c) 2022, NVIDIA CORPORATION.

./build.sh libraft tests bench -v --allgpuarch --compile-dist --compile-nn
14 changes: 14 additions & 0 deletions conda/recipes/libraft/conda_build_config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
cmake_version:
- ">=3.20.1,!=3.23.0"

nccl_version:
- ">=2.9.9"

gtest_version:
- "=1.10.0"

libcusolver_version:
- ">=11.2.1"

libfaiss_version:
- "1.7.0 *_cuda"
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env bash
# Copyright (c) 2022, NVIDIA CORPORATION.

./build.sh libraft --install -v --allgpuarch --compile-nn
cmake --install cpp/build --component raft_distance
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env bash
# Copyright (c) 2022, NVIDIA CORPORATION.

./build.sh libraft --install -v --allgpuarch
cmake --install cpp/build --component raft
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env bash
# Copyright (c) 2022, NVIDIA CORPORATION.

./build.sh libraft --install -v --allgpuarch --compile-dist
cmake --install cpp/build --component raft_nn
4 changes: 4 additions & 0 deletions conda/recipes/libraft/install_libraft_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Copyright (c) 2022, NVIDIA CORPORATION.

cmake --install cpp/build --component testing
136 changes: 136 additions & 0 deletions conda/recipes/libraft/meta.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
# Copyright (c) 2022, NVIDIA CORPORATION.

# Usage:
# conda build . -c conda-forge -c nvidia -c rapidsai
{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %}
{% set cuda_version = '.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %}
{% set cuda_major = cuda_version.split('.')[0] %}
{% set ucx_py_version = environ.get('UCX_PY_VERSION') %}
{% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0

package:
name: libraft-split

source:
git_url: ../../..

build:
script_env:
- CC
- CXX
- CUDAHOSTCXX
- PARALLEL_LEVEL
- VERSION_SUFFIX
- PROJECT_FLASH
- CMAKE_GENERATOR
- CMAKE_C_COMPILER_LAUNCHER
- CMAKE_CXX_COMPILER_LAUNCHER
- CMAKE_CUDA_COMPILER_LAUNCHER
- SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64]
- SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64]
- SCCACHE_BUCKET=rapids-sccache
- SCCACHE_REGION=us-west-2
- SCCACHE_IDLE_TIMEOUT=32768

requirements:
build:
- cmake {{ cmake_version }}
host:
- cudatoolkit {{ cuda_version }}.*
- faiss-proc=*=cuda
- gmock {{ gtest_version }}
- gtest {{ gtest_version }}
- lapack
- libcusolver {{ libcusolver_version }}
- libfaiss {{ libfaiss_version }}
- librmm {{ minor_version }}
- nccl {{ nccl_version }}
- ucx-proc=*=gpu
- ucx-py {{ ucx_py_version }}

outputs:
- name: libraft-headers
version: {{ version }}
script: install_libraft_headers.sh
build:
number: {{ GIT_DESCRIBE_NUMBER }}
string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
requirements:
build:
- cmake {{ cmake_version }}
run:
- cudatoolkit {{ cuda_spec }}
- libcusolver {{ libcusolver_version }}
- librmm {{ minor_version }}
- nccl {{ nccl_version }}
- ucx-proc=*=gpu
- ucx-py {{ ucx_py_version }}
about:
home: http://rapids.ai/
license: Apache-2.0
summary: libraft-headers library
- name: libraft-distance
version: {{ version }}
script: install_libraft_distance.sh
build:
number: {{ GIT_DESCRIBE_NUMBER }}
string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
requirements:
build:
- cmake {{ cmake_version }}
host:
- {{ pin_subpackage('libraft-headers', exact=True) }}
run:
- cudatoolkit {{ cuda_spec }}
- libcusolver {{ libcusolver_version }}
- librmm {{ minor_version }}
- nccl {{ nccl_version }}
- ucx-proc=*=gpu
- ucx-py {{ ucx_py_version }}
- {{ pin_subpackage('libraft-headers', exact=True) }}
about:
home: http://rapids.ai/
license: Apache-2.0
summary: libraft-distance library
- name: libraft-nn
version: {{ version }}
script: install_libraft_nn.sh
build:
number: {{ GIT_DESCRIBE_NUMBER }}
string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
requirements:
build:
- cmake {{ cmake_version }}
host:
- {{ pin_subpackage('libraft-headers', exact=True) }}
run:
- cudatoolkit {{ cuda_spec }}
- faiss-proc=*=cuda
- libcusolver {{ libcusolver_version }}
- libfaiss {{ libfaiss_version }}
- librmm {{ minor_version }}
- {{ pin_subpackage('libraft-headers', exact=True) }}
about:
home: http://rapids.ai/
license: Apache-2.0
summary: libraft-nn library
- name: libraft-tests
version: {{ version }}
script: install_libraft_tests.sh
build:
number: {{ GIT_DESCRIBE_NUMBER }}
string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
requirements:
build:
- cmake {{ cmake_version }}
run:
- gmock {{ gtest_version }}
- gtest {{ gtest_version }}
- {{ pin_subpackage('libraft-distance', exact=True) }}
- {{ pin_subpackage('libraft-headers', exact=True) }}
- {{ pin_subpackage('libraft-nn', exact=True) }}
about:
home: http://rapids.ai/
license: Apache-2.0
summary: libraft tests
Loading

0 comments on commit 8ad57f5

Please sign in to comment.