diff --git a/.github/labeler.yml b/.github/labeler.yml index 112d0d2f9..3260eeae2 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -17,5 +17,5 @@ cpp: - 'tests/**' - 'doxygen/**' -gpuCI: +ci: - 'ci/**' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd654bef3..61ac5873a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,10 +56,5 @@ contributing to. Start with _Step 3_ from above, commenting on the issue to let others know you are working on it. If you have any questions related to the implementation of the issue, ask them in the issue instead of the PR. -### Building and Testing on a gpuCI image locally - -Before submitting a pull request, you can do a local build and test on your machine that mimics our gpuCI environment using the `ci/local/build.sh` script. -For detailed information on usage of this script, see [here](ci/local/README.md). - ## Attribution Portions adopted from https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md diff --git a/README.md b/README.md index 2292b331e..ef2fa43a1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ #
 RMM: RAPIDS Memory Manager
-[![Build Status](https://gpuci.gpuopenanalytics.com/job/rapidsai/job/gpuci/job/rmm/job/branches/job/rmm-branch-pipeline/badge/icon)](https://gpuci.gpuopenanalytics.com/job/rapidsai/job/gpuci/job/rmm/job/branches/job/rmm-branch-pipeline/) - **NOTE:** For the latest stable [README.md](https://github.com/rapidsai/rmm/blob/main/README.md) ensure you are on the `main` branch. ## Resources diff --git a/ci/checks/changelog.sh b/ci/checks/changelog.sh deleted file mode 100755 index 2cb0edbac..000000000 --- a/ci/checks/changelog.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020, NVIDIA CORPORATION. -######################### -# RMM CHANGELOG Tester # -######################### - -# Checkout main for comparison -git checkout --force --quiet main - -# Switch back to tip of PR branch -git checkout --force --quiet current-pr-branch - -# Ignore errors during searching -set +e - -# Get list of modified files between matster and PR branch -CHANGELOG=`git diff --name-only main...current-pr-branch | grep CHANGELOG.md` -# Check if CHANGELOG has PR ID -PRNUM=`cat CHANGELOG.md | grep "$PR_ID"` -RETVAL=0 - -# Return status of check result -if [ "$CHANGELOG" != "" -a "$PRNUM" != "" ] ; then - echo -e "\n\n>>>> PASSED: CHANGELOG.md has been updated with current PR information.\n\nPlease ensure the update meets the following criteria.\n" -else - echo -e "\n\n>>>> FAILED: CHANGELOG.md has not been updated!\n\nPlease add a line describing this PR to CHANGELOG.md in the repository root directory. The line should meet the following criteria.\n" - RETVAL=1 -fi - -cat << EOF - It should be placed under the section for the appropriate release. - It should be placed under "New Features", "Improvements", or "Bug Fixes" as appropriate. - It should be formatted as '- PR # ' - Example format for #491 '- PR #491 Add CI test script to check for updates to CHANGELOG.md in PRs' - - -EOF - -exit $RETVAL diff --git a/ci/checks/style.sh b/ci/checks/style.sh deleted file mode 100644 index cec606871..000000000 --- a/ci/checks/style.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020, NVIDIA CORPORATION. -##################### -# RMM Style Tester # -##################### - -# Ignore errors and set path -set +e -PATH=/opt/conda/bin:$PATH -LC_ALL=C.UTF-8 -LANG=C.UTF-8 - -# Activate common conda env -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -# Run isort and get results/return code -ISORT=`isort --check-only python --settings-path=python/setup.cfg ` -ISORT_RETVAL=$? - -# Run black and get results/return code -BLACK=`black --config python/pyproject.toml --check python` -BLACK_RETVAL=$? - -# Run flake8 and get results/return code -FLAKE=`flake8 --config=python/.flake8 python` -FLAKE_RETVAL=$? - -# Run flake8-cython and get results/return code -FLAKE_CYTHON=`flake8 --config=python/.flake8.cython` -FLAKE_CYTHON_RETVAL=$? - -# Run clang-format and check for a consistent code format -CLANG_FORMAT=`python scripts/run-clang-format.py 2>&1` -CLANG_FORMAT_RETVAL=$? - -# Run cmake-format / cmake-lint and get results/return code -CMAKE_FILES=(`find | grep -E "^.*\.cmake(\.in)?$|^.*/CMakeLists.txt$"`) - -CMAKE_FORMATS=() -CMAKE_FORMAT_RETVAL=0 - -CMAKE_LINTS=() -CMAKE_LINT_RETVAL=0 - -CURRENT_TAG=$(git tag --merged HEAD | grep -xE '^v.*' | sort --version-sort | tail -n 1 | tr -d 'v') -CURRENT_MAJOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[1]}') -CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}') -CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} -gpuci_retry curl -s https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${CURRENT_SHORT_TAG}/cmake-format-rapids-cmake.json -o cmake/rapids-cmake.json - -for cmake_file in "${CMAKE_FILES[@]}"; do - cmake-format --in-place --config-files cmake/config.json cmake/rapids-cmake.json -- ${cmake_file} - TMP_CMAKE_FORMAT=`git diff --color --exit-code -- ${cmake_file}` - TMP_CMAKE_FORMAT_RETVAL=$? - if [ "$TMP_CMAKE_FORMAT_RETVAL" != "0" ]; then - CMAKE_FORMAT_RETVAL=1 - CMAKE_FORMATS+=("$TMP_CMAKE_FORMAT") - fi - - TMP_CMAKE_LINT=`cmake-lint --config-files cmake/config.json cmake/rapids-cmake.json -- ${cmake_file}` - TMP_CMAKE_LINT_RETVAL=$? - if [ "$TMP_CMAKE_LINT_RETVAL" != "0" ]; then - CMAKE_LINT_RETVAL=1 - CMAKE_LINTS+=("$TMP_CMAKE_LINT") - fi -done - - -# Output results if failure otherwise show pass -if [ "$ISORT_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n" - echo -e "$ISORT" - echo -e "\n\n>>>> FAILED: isort style check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: isort style check\n\n" -fi - -if [ "$BLACK_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: black style check; begin output\n\n" - echo -e "$BLACK" - echo -e "\n\n>>>> FAILED: black style check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: black style check\n\n" -fi - -if [ "$FLAKE_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n" - echo -e "$FLAKE" - echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: flake8 style check\n\n" -fi - -if [ "$FLAKE_CYTHON_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: flake8-cython style check; begin output\n\n" - echo -e "$FLAKE_CYTHON" - echo -e "\n\n>>>> FAILED: flake8-cython style check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: flake8-cython style check\n\n" -fi - -if [ "$CLANG_FORMAT_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: clang format check; begin output\n\n" - echo -e "$CLANG_FORMAT" - echo -e "\n\n>>>> FAILED: clang format check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: clang format check\n\n" -fi - -if [ "$CMAKE_FORMAT_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: cmake format check; begin output\n\n" - for CMAKE_FORMAT in "${CMAKE_FORMATS[@]}"; do - echo -e "$CMAKE_FORMAT" - echo -e "\n" - done - echo -e "\n\n>>>> FAILED: cmake format check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: cmake format check\n\n" -fi - -if [ "$CMAKE_LINT_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: cmake lint check; begin output\n\n" - for CMAKE_LINT in "${CMAKE_LINTS[@]}"; do - echo -e "$CMAKE_LINT" - echo -e "\n" - done - echo -e "\n\n>>>> FAILED: cmake lint check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: cmake lint check\n\n" -fi - -RETVALS=($ISORT_RETVAL $BLACK_RETVAL $FLAKE_RETVAL $FLAKE_CYTHON_RETVAL $CLANG_FORMAT_RETVAL $CMAKE_FORMAT_RETVAL $CMAKE_LINT_RETVAL) -IFS=$'\n' -RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1` - -exit $RETVAL diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh deleted file mode 100755 index 747930924..000000000 --- a/ci/cpu/build.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2020, NVIDIA CORPORATION. -###################################### -# rmm CPU build script for CI # -###################################### -set -e - -# Set path, build parallel level and build generator -export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} -export CMAKE_GENERATOR="Ninja" -export CONDA_BLD_DIR="$WORKSPACE/.conda-bld" - -# Set home to the job's workspace -export HOME=$WORKSPACE - -# Switch to project root; also root of repo checkout -cd $WORKSPACE - -# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts) -export GPUCI_CONDA_RETRY_MAX=1 -export GPUCI_CONDA_RETRY_SLEEP=30 - -# Workaround to keep Jenkins builds working -# until we migrate fully to GitHub Actions -export RAPIDS_CUDA_VERSION="${CUDA}" - -# If nightly build, append current YYMMDD to version -if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then - export VERSION_SUFFIX=`date +%y%m%d` -fi - -################################################################################ -# SETUP - Check environment -################################################################################ - -gpuci_logger "Get env" -env - -gpuci_logger "Activate conda env" -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -# Remove rapidsai-nightly channel if we are building main branch -if [ "$SOURCE_BRANCH" = "main" ]; then - conda config --system --remove channels rapidsai-nightly -fi - -gpuci_logger "Check versions" -python --version -$CC --version -$CXX --version - -gpuci_logger "Check conda environment" -conda info -conda config --show-sources -conda list --show-channel-urls - -# FIX Added to deal with Anancoda SSL verification issues during conda builds -conda config --set ssl_verify False - -# FIXME: Move installation to gpuci/rapidsai images -gpuci_mamba_retry install -c conda-forge boa - -################################################################################ -# BUILD - Conda package builds (conda deps: librmm <- rmm) -################################################################################ - -if [[ "$BUILD_LIBRMM" == "1" ]]; then - gpuci_logger "Build conda pkg for librmm" - if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - gpuci_conda_retry mambabuild conda/recipes/librmm --python=$PYTHON - else - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/librmm - mkdir -p ${CONDA_BLD_DIR}/librmm - mv ${CONDA_BLD_DIR}/work/ ${CONDA_BLD_DIR}/librmm/work - fi - gpuci_logger "sccache stats" - sccache --show-stats -fi - -if [[ "$BUILD_RMM" == "1" ]]; then - gpuci_logger "Build conda pkg for rmm" - if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - gpuci_conda_retry mambabuild conda/recipes/rmm --python=$PYTHON - else - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir \ - -c $WORKSPACE/ci/artifacts/rmm/cpu/.conda-bld/ conda/recipes/rmm - mkdir -p ${CONDA_BLD_DIR}/rmm - mv ${CONDA_BLD_DIR}/work/ ${CONDA_BLD_DIR}/rmm/work - fi -fi - -################################################################################ -# UPLOAD - Conda packages -################################################################################ - -# gpuci_logger "Upload conda packages" -# source ci/cpu/upload.sh diff --git a/ci/cpu/prebuild.sh b/ci/cpu/prebuild.sh deleted file mode 100644 index e434ed550..000000000 --- a/ci/cpu/prebuild.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -#Always upload RMM packages -export UPLOAD_RMM=1 -export UPLOAD_LIBRMM=1 - -if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - #If project flash is not activate, always build both - export BUILD_LIBRMM=1 - export BUILD_RMM=1 -fi diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh deleted file mode 100644 index 9bff2db31..000000000 --- a/ci/cpu/upload.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# -# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh - -set -e - -# Setup 'gpuci_retry' for upload retries (results in 4 total attempts) -export GPUCI_RETRY_MAX=3 -export GPUCI_RETRY_SLEEP=30 - -# Set default label options if they are not defined elsewhere -export LABEL_OPTION=${LABEL_OPTION:-"--label main"} - -# Skip uploads unless BUILD_MODE == "branch" -if [ ${BUILD_MODE} != "branch" ]; then - echo "Skipping upload" - return 0 -fi - -# Skip uploads if there is no upload key -if [ -z "$MY_UPLOAD_KEY" ]; then - echo "No upload key" - return 0 -fi - - -################################################################################ -# UPLOAD - Conda packages -################################################################################ - -gpuci_logger "Starting conda uploads" - -if [[ "$BUILD_LIBRMM" == "1" && "$UPLOAD_LIBRMM" == "1" ]]; then - export LIBRMM_FILES=$(conda build --croot ${CONDA_BLD_DIR} conda/recipes/librmm --output) - while read -r LIBRMM_FILE; do - test -e ${LIBRMM_FILE} - echo "Upload librmm file: ${LIBRMM_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRMM_FILE} --no-progress - done <<< "${LIBRMM_FILES}" -fi - -if [[ "$BUILD_RMM" == "1" && "$UPLOAD_RMM" == "1" ]]; then - export RMM_FILES=$(conda build --croot ${CONDA_BLD_DIR} conda/recipes/rmm --python=$PYTHON --output) - while read -r RMM_FILE; do - test -e ${RMM_FILE} - echo "Upload rmm file: ${RMM_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${RMM_FILE} --no-progress - done <<< "${RMM_FILES}" -fi - diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh deleted file mode 100755 index 3db5d3f77..000000000 --- a/ci/gpu/build.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2020, NVIDIA CORPORATION. -###################################### -# rmm GPU build & testscript for CI # -###################################### -set -e -NUMARGS=$# -ARGS=$* - -# Arg parsing function -function hasArg { - (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") -} - -# Set path and build parallel level -export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} - -# Set home to the job's workspace -export HOME=$WORKSPACE - -# Switch to project root; also root of repo checkout -cd $WORKSPACE - -# Determine CUDA release version -export CUDA_MAJOR_VER=$(echo "${CUDA_VERSION}" | cut -f 1 -d.) -export CUDA_MINOR_VER=$(echo "${CUDA_VERSION}" | cut -f 2 -d.) -export CUDA_REL="${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}" - -# Workaround to keep Jenkins builds working -# until we migrate fully to GitHub Actions -export RAPIDS_CUDA_VERSION="${CUDA}" - -# Get latest tag and number of commits since tag -export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags` -export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count` -export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` -unset GIT_DESCRIBE_TAG - -################################################################################ -# SETUP - Check environment -################################################################################ - -gpuci_logger "Get env" -env - -gpuci_logger "Activate conda env" -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -gpuci_logger "Check versions" -python --version -$CC --version -$CXX --version - -gpuci_logger "Check conda environment" -conda info -conda config --show-sources -conda list --show-channel-urls - -if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - # Install build env - gpuci_mamba_retry install -y \ - "cudatoolkit=$CUDA_REL" \ - "rapids-build-env=${MINOR_VERSION}.*" - - # https://docs.rapids.ai/maintainers/depmgmt/ - # conda remove --force rapids-build-env - # gpuci_mamba_retry install "your-pkg=1.0.0" - - ################################################################################ - # BUILD - Build and install librmm and rmm - ################################################################################ - - gpuci_logger "Build and install librmm and rmm" - "$WORKSPACE/build.sh" -v clean librmm rmm benchmarks tests - - ################################################################################ - # Test - librmm - ################################################################################ - - if hasArg --skip-tests; then - gpuci_logger "Skipping Tests" - else - gpuci_logger "Check GPU usage" - nvidia-smi - - gpuci_logger "Running googletests" - - cd "${WORKSPACE}/build" - GTEST_OUTPUT="xml:${WORKSPACE}/test-results/" make -j${PARALLEL_LEVEL} test - - gpuci_logger "Running rmm pytests..." - cd $WORKSPACE/python - py.test --cache-clear --basetemp=${WORKSPACE}/rmm-cuda-tmp --junitxml=${WORKSPACE}/test-results/junit-rmm.xml -v --cov-config=.coveragerc --cov=rmm --cov-report=xml:${WORKSPACE}/python/rmm-coverage.xml --cov-report term - fi -else - gpuci_mamba_retry install -c $WORKSPACE/ci/artifacts/rmm/cpu/.conda-bld/ librmm librmm-tests - - TESTRESULTS_DIR=${WORKSPACE}/test-results - mkdir -p ${TESTRESULTS_DIR} - SUITEERROR=0 - - gpuci_logger "Check GPU usage" - nvidia-smi - - gpuci_logger "Running googletests" - # run gtests from librmm-tests package - for gt in "$CONDA_PREFIX/bin/gtests/librmm/"* ; do - ${gt} --gtest_output=xml:${TESTRESULTS_DIR}/ - exitcode=$? - if (( ${exitcode} != 0 )); then - SUITEERROR=${exitcode} - echo "FAILED: GTest ${gt}" - fi - done - - # TODO: Move boa install to gpuci/rapidsai - gpuci_mamba_retry install boa - export CONDA_BLD_DIR="$WORKSPACE/.conda-bld" - gpuci_logger "Building and installing rmm" - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} \ - -c $WORKSPACE/ci/artifacts/rmm/cpu/.conda-bld/ conda/recipes/rmm - gpuci_mamba_retry install -c $WORKSPACE/ci/artifacts/rmm/cpu/.conda-bld/ \ - -c ${CONDA_BLD_DIR} rmm - - cd $WORKSPACE/python - gpuci_logger "pytest rmm" - py.test --cache-clear --junitxml=${WORKSPACE}/test-results/junit-rmm.xml -v --cov-config=.coveragerc --cov=rmm --cov-report=xml:${WORKSPACE}/python/rmm-coverage.xml --cov-report term - exitcode=$? - if (( ${exitcode} != 0 )); then - SUITEERROR=${exitcode} - echo "FAILED: 1 or more tests in /rmm/python" - fi - - exit ${SUITEERROR} -fi diff --git a/ci/local/README.md b/ci/local/README.md deleted file mode 100644 index e37268f04..000000000 --- a/ci/local/README.md +++ /dev/null @@ -1,58 +0,0 @@ -## Purpose - -This script is designed for developer and contributor use. This tool mimics the actions of gpuCI on your local machine. This allows you to test and even debug your code inside a gpuCI base container before pushing your code as a GitHub commit. -The script can be helpful in locally triaging and debugging RAPIDS continuous integration failures. - -## Requirements - -``` -nvidia-docker -``` - -## Usage - -``` -bash build.sh [-h] [-H] [-s] [-r ] [-i ] -Build and test your local repository using a base gpuCI Docker image - -where: - -H Show this help text - -r Path to repository (defaults to working directory) - -i Use Docker image (default is gpuci/rapidsai:${NIGHTLY_VERSION}-cuda10.1-devel-ubuntu16.04-py3.7) - -s Skip building and testing and start an interactive shell in a container of the Docker image -``` - -Example Usage: -`bash build.sh -r ~/rapids/rmm -i gpuci/rapidsai:0.16-cuda10.2-devel-ubuntu16.04-py3.7` - -For a full list of available gpuCI docker images, visit our [DockerHub](https://hub.docker.com/r/gpuci/rapidsai/tags) page. - -Style Check: -```bash -$ bash ci/local/build.sh -r ~/rapids/rmm -s -$ . /opt/conda/etc/profile.d/conda.sh -$ conda activate rapids # Activate gpuCI conda environment -$ cd rapids -$ flake8 python -``` - -## Information - -There are some caveats to be aware of when using this script, especially if you plan on developing from within the container itself. - - -### Docker Image Build Repository - -The docker image will generate build artifacts in a folder on your machine located in the `root` directory of the repository you passed to the script. For the above example, the directory is named `~/rapids/rmm/build_rapidsai_cuda10.1-ubuntu16.04-py3.7/`. Feel free to remove this directory after the script is finished. - -*Note*: The script *will not* override your local build repository. Your local environment stays in tact. - - -### Where The User is Dumped - -The script will build your repository and run all tests. If any tests fail, it dumps the user into the docker container itself to allow you to debug from within the container. If all the tests pass as expected the container exits and is automatically removed. Remember to exit the container if tests fail and you do not wish to debug within the container itself. - - -### Container File Structure - -Your repository will be located in the `/rapids/` folder of the container. This folder is volume mounted from the local machine. Any changes to the code in this repository are replicated onto the local machine. The `cpp/build` and `python/build` directories within your repository is on a separate mount to avoid conflicting with your local build artifacts. diff --git a/ci/local/build.sh b/ci/local/build.sh deleted file mode 100755 index 7adf8e62f..000000000 --- a/ci/local/build.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash - -GIT_DESCRIBE_TAG=`git describe --tags` -MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` - -DOCKER_IMAGE="gpuci/rapidsai:${MINOR_VERSION}-cuda10.1-devel-ubuntu16.04-py3.7" -REPO_PATH=${PWD} -RAPIDS_DIR_IN_CONTAINER="/rapids" -CPP_BUILD_DIR="build" -PYTHON_BUILD_DIR="python/build" -CONTAINER_SHELL_ONLY=0 - -SHORTHELP="$(basename "$0") [-h] [-H] [-s] [-r ] [-i ]" -LONGHELP="${SHORTHELP} -Build and test your local repository using a base gpuCI Docker image - -where: - -H Show this help text - -r Path to repository (defaults to working directory) - -i Use Docker image (default is ${DOCKER_IMAGE}) - -s Skip building and testing and start an interactive shell in a container of the Docker image -" - -# Limit GPUs available to container based on CUDA_VISIBLE_DEVICES -if [[ -z "${CUDA_VISIBLE_DEVICES}" ]]; then - NVIDIA_VISIBLE_DEVICES="all" -else - NVIDIA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} -fi - -while getopts ":hHr:i:s" option; do - case ${option} in - r) - REPO_PATH=${OPTARG} - ;; - i) - DOCKER_IMAGE=${OPTARG} - ;; - s) - CONTAINER_SHELL_ONLY=1 - ;; - h) - echo "${SHORTHELP}" - exit 0 - ;; - H) - echo "${LONGHELP}" - exit 0 - ;; - *) - echo "ERROR: Invalid flag" - echo "${SHORTHELP}" - exit 1 - ;; - esac -done - -REPO_PATH_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")" -CPP_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")/${CPP_BUILD_DIR}" -PYTHON_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")/${PYTHON_BUILD_DIR}" - - -# BASE_CONTAINER_BUILD_DIR is named after the image name, allowing for -# multiple image builds to coexist on the local filesystem. This will -# be mapped to the typical BUILD_DIR inside of the container. Builds -# running in the container generate build artifacts just as they would -# in a bare-metal environment, and the host filesystem is able to -# maintain the host build in BUILD_DIR as well. - -# FIXME: Fix the shellcheck complaints -# shellcheck disable=SC2001,SC2005 -BASE_CONTAINER_BUILD_DIR=${REPO_PATH}/build_$(echo "$(basename "${DOCKER_IMAGE}")"|sed -e 's/:/_/g') -CPP_CONTAINER_BUILD_DIR=${BASE_CONTAINER_BUILD_DIR}/cpp -PYTHON_CONTAINER_BUILD_DIR=${BASE_CONTAINER_BUILD_DIR}/python - -# Note the prevention of expanding variables. Preventing expansion is critical as -# expansion in context of calling shell would lead to incorrect result as not -# all variables are set -BUILD_SCRIPT="#!/bin/bash -set -e -WORKSPACE=${REPO_PATH_IN_CONTAINER} -PREBUILD_SCRIPT=${REPO_PATH_IN_CONTAINER}/ci/gpu/prebuild.sh -BUILD_SCRIPT=${REPO_PATH_IN_CONTAINER}/ci/gpu/build.sh -cd \${WORKSPACE} -if [ -f \${PREBUILD_SCRIPT} ]; then - source \${PREBUILD_SCRIPT} -fi -yes | source \${BUILD_SCRIPT} -" - -if (( CONTAINER_SHELL_ONLY == 0 )); then - COMMAND="${CPP_BUILD_DIR_IN_CONTAINER}/build.sh || bash" -else - COMMAND="bash" -fi - -# Create the build dir for the container to mount, generate the build script inside of it -mkdir -p "${BASE_CONTAINER_BUILD_DIR}" -mkdir -p "${CPP_CONTAINER_BUILD_DIR}" -mkdir -p "${PYTHON_CONTAINER_BUILD_DIR}" -# Create build directories. This is to ensure correct owner for directories. If -# directories don't exist there is side effect from docker volume mounting creating build -# directories owned by root(volume mount point(s)) -mkdir -p "${REPO_PATH}/${CPP_BUILD_DIR}" -mkdir -p "${REPO_PATH}/${PYTHON_BUILD_DIR}" - -echo "${BUILD_SCRIPT}" > "${CPP_CONTAINER_BUILD_DIR}/build.sh" -chmod ugo+x "${CPP_CONTAINER_BUILD_DIR}/build.sh" - -# Mount passwd and group files to docker. This allows docker to resolve username and group -# avoiding these nags: -# * groups: cannot find name for group ID ID -# * I have no name!@id:/$ -# For ldap user user information is not present in system /etc/passwd and /etc/group files. -# Hence we generate dummy files for ldap users which docker uses to resolve username and group - -PASSWD_FILE="/etc/passwd" -GROUP_FILE="/etc/group" - -USER_FOUND=$(grep -wc "$(whoami)" < "$PASSWD_FILE") -if [ "$USER_FOUND" == 0 ]; then - echo "Local User not found, LDAP WAR for docker mounts activated. Creating dummy passwd and group" - echo "files to allow docker resolve username and group" - cp "$PASSWD_FILE" /tmp/passwd - PASSWD_FILE="/tmp/passwd" - cp "$GROUP_FILE" /tmp/group - GROUP_FILE="/tmp/group" - echo "$(whoami):x:$(id -u):$(id -g):$(whoami),,,:$HOME:$SHELL" >> "$PASSWD_FILE" - echo "$(whoami):x:$(id -g):" >> "$GROUP_FILE" -fi - - -# Run the generated build script in a container -docker pull "${DOCKER_IMAGE}" - -DOCKER_MAJOR=$(docker -v|sed 's/[^[0-9]*\([0-9]*\).*/\1/') -GPU_OPTS="--gpus device=${NVIDIA_VISIBLE_DEVICES}" -if [ "$DOCKER_MAJOR" -lt 19 ] -then - GPU_OPTS="--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES='${NVIDIA_VISIBLE_DEVICES}'" -fi - -docker run --rm -it ${GPU_OPTS} \ - --user "$(id -u)":"$(id -g)" \ - -v "${REPO_PATH}":"${REPO_PATH_IN_CONTAINER}" \ - -v "${CPP_CONTAINER_BUILD_DIR}":"${CPP_BUILD_DIR_IN_CONTAINER}" \ - -v "${PYTHON_CONTAINER_BUILD_DIR}":"${PYTHON_BUILD_DIR_IN_CONTAINER}" \ - -v "$PASSWD_FILE":/etc/passwd:ro \ - -v "$GROUP_FILE":/etc/group:ro \ - --cap-add=SYS_PTRACE \ - "${DOCKER_IMAGE}" bash -c "${COMMAND}"