Permalink
Browse files

migrate DEBUG & ASAN to use circle ci

  • Loading branch information...
zrphercule authored and rdzhabarov committed Oct 18, 2018
1 parent 55c6229 commit 6fd742554946fc295d826843b2f444bc3c937d6b
Showing with 219 additions and 76 deletions.
  1. +67 −0 .circleci/build.sh
  2. +81 −3 .circleci/config.yml
  3. +71 −0 .circleci/test.sh
  4. +0 −23 .travis.yml
  5. +0 −50 .travis/run_tests.sh
@@ -0,0 +1,67 @@
#!/bin/bash

# This script setup an working environemnt for running glow tests and gtest driver.
# By default, we run with the enabled CPU backend and disabled OpenCL backend.
set -ex

export MAX_JOBS=8

# setup sccache wrappers
if hash sccache 2>/dev/null; then
SCCACHE_BIN_DIR="/tmp/sccache"
mkdir -p "$SCCACHE_BIN_DIR"
for compiler in cc c++ gcc g++ x86_64-linux-gnu-gcc; do
(
echo "#!/bin/sh"
echo "exec $(which sccache) $(which $compiler) \"\$@\""
) > "$SCCACHE_BIN_DIR/$compiler"
chmod +x "$SCCACHE_BIN_DIR/$compiler"
done
export PATH="$SCCACHE_BIN_DIR:$PATH"
fi

GLOW_DIR=$PWD

# Install Glow dependencies
sudo apt-get update
sudo apt-get install -y llvm-6.0 llvm-6.0-dev libpng-dev

# Redirect clang
sudo ln -s /usr/bin/clang-6.0 /usr/bin/clang
sudo ln -s /usr/bin/clang++-6.0 /usr/bin/clang++

# Install ninja and (newest version of) cmake through pip
sudo pip install ninja cmake
hash cmake ninja

# Build glow
cd ${GLOW_DIR}
mkdir build && cd build
CMAKE_ARGS=()
CMAKE_ARGS+=("-DGLOW_WITH_OPENCL=OFF")
CMAKE_ARGS+=("-DGLOW_WITH_CPU=ON")
if [[ "$CIRCLE_JOB" == ASAN ]]; then
CMAKE_ARGS+=("-DGLOW_USE_SANITIZER='Address;Undefined'")
fi
if [[ "$CIRCLE_JOB" == DEBUG ]]; then
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug")
else
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
fi
cmake -GNinja ${CMAKE_ARGS[*]} ../
ninja

# Build onnxifi test driver (Only for DEBUG mode)
if [[ "$CIRCLE_JOB" == DEBUG ]]; then
ONNX_DIR="${GLOW_DIR}/thirdparty/onnx"
cd ${ONNX_DIR}
mkdir build_onnx && cd build_onnx
cmake -GNinja -DONNX_BUILD_TESTS=ON -DONNXIFI_DUMMY_BACKEND=OFF ../
ninja onnxifi_test_driver_gtests
cp ${ONNX_DIR}/build_onnx/onnxifi_test_driver_gtests ${GLOW_DIR}/build
fi

# Report sccache hit/miss stats
if hash sccache 2>/dev/null; then
sccache --show-stats
fi
@@ -1,7 +1,85 @@
version: 2

# NOTE: We only perform the merge in build step and not in test step, because
# all source files will be shared from build to test
merge_with_master: &merge_with_master
name: Merge Onto Master
command: |
if [[ "${CIRCLE_BRANCH}" != "master" ]]; then
git config --global user.email "circleci.ossci@gmail.com"
git config --global user.name "CircleCI"
git config remote.origin.url "$CIRCLE_REPOSITORY_URL"
git config --add remote.origin.fetch +refs/heads/master:refs/remotes/origin/master
git fetch --tags --progress "$CIRCLE_REPOSITORY_URL" +refs/heads/master:refs/remotes/origin/master --depth=50 --quiet
export GIT_MERGE_TARGET=`git log -n 1 --pretty=format:"%H" origin/master`
echo "GIT_MERGE_TARGET: " ${GIT_MERGE_TARGET}
export GIT_COMMIT=${CIRCLE_SHA1}
echo "GIT_COMMIT: " ${GIT_COMMIT}
git checkout -f ${GIT_COMMIT}
git reset --hard ${GIT_COMMIT}
git merge --no-edit --no-ff ${GIT_MERGE_TARGET}
fi
update_submodule: &update_submodule
name: Update Submodule
command: |
git submodule sync
git submodule update --recursive --init
linux_default: &linux_default
resource_class: large
machine:
image: default
steps:
- checkout
- run:
<<: *merge_with_master
- run:
<<: *update_submodule
- run:
name: Launch Docker Container
no_output_timeout: "1h"
command: |
set -e
export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_FOR_ECR_READ_WRITE}
export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY_FOR_ECR_READ_WRITE}
sudo pip install awscli==1.16.35 -qqq
eval $(aws ecr get-login --region us-east-1 --no-include-email)
docker pull ${DOCKER_IMAGE}
sudo pkill -SIGHUP dockerd
WORKDIR=/var/lib/jenkins/workspace
pid=$(docker run -t -d -w $WORKDIR ${DOCKER_IMAGE})
docker cp /home/circleci/project/. "$pid:$WORKDIR"
docker exec -u jenkins ${pid} sudo chown -R jenkins ${WORKDIR}
echo ${pid} > .docker_pid
- run:
name: Build
no_output_timeout: "1h"
command: |
docker exec -e CIRCLE_JOB=${CIRCLE_JOB} -e SCCACHE_BUCKET=${SCCACHE_BUCKET} -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET} -u jenkins $(cat .docker_pid) .circleci/build.sh
- run:
name: Test
no_output_timeout: "1h"
command: |
docker exec -e CIRCLE_JOB=${CIRCLE_JOB} -u jenkins $(cat .docker_pid) .circleci/test.sh
jobs:
DEBUG:
environment:
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang6.0-ubuntu16.04:213"
<<: *linux_default
ASAN:
environment:
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang6.0-ubuntu16.04:213"
<<: *linux_default
workflows:
version: 2
build:
steps:
- checkout
machine: true
jobs:
- DEBUG
- ASAN
@@ -0,0 +1,71 @@
#!/bin/bash

# This script runs all tests in glow, including onnxifi gtests


set -euxo pipefail

GLOW_DIR=$PWD
GLOW_BUILD_DIR=${GLOW_DIR}/build
cd ${GLOW_BUILD_DIR}

TEST_NAME=$CIRCLE_JOB

# Pass in which tests to run (one of {test, test_unopt}).
run_unit_tests() {
CTEST_PARALLEL_LEVEL=4 ninja ${1} || ( cat Testing/Temporary/LastTest.log && exit 1 )
}

# Pass one of {YES, NO} for QUANTIZE.
run_and_check_bundle() {
echo "Checking lenet_mnist bundle with QUANTIZE=${1}"
cd ${GLOW_SRC}/examples/bundles/lenet_mnist/
( QUANTIZE=${1} make &> raw_results.txt ) || ( cat raw_results.txt && exit 1 )
( tail -n72 raw_results.txt | grep -F "Result: " > results.txt ) || ( cat raw_results.txt && exit 1 )
diff results.txt ${GLOW_SRC}/.ci/lenet_mnist_expected_output.txt
rm results.txt raw_results.txt
echo "Successfully completed checking lenet_mnist bundle with QUANTIZE=${1}"
}

export GLOW_SRC=${PWD}/..
export LOADER=${GLOW_SRC}/build/bin/image-classifier

case ${TEST_NAME} in
ASAN)
run_unit_tests test
;;

DEBUG)
run_unit_tests test
run_unit_tests test_unopt
run_and_check_bundle YES
run_and_check_bundle NO
;;

*)
echo "Error, '${TEST_NAME}' not valid mode; Must be one of {ASAN, DEBUG}."
exit 1
;;
esac


# Run ONNX test
ONNX_DIR="${GLOW_DIR}/thirdparty/onnx"
# ONNX test data dir
TESTDATA_DIR="${ONNX_DIR}/onnx/backend/test/data/node"

# Asan is not enbaled in onnx, therefore we should skip it for now.
# TODO: Enable asan test. Rui Zhu.
if [[ "$CIRCLE_JOB" != ASAN ]]; then
# Banned known buggy test cases from gtest
CRASHED_TEST_CASES="*test_softmax_axis_0*:*test_batchnorm_epsilon*:*test_batchnorm_example*:*test_sum_example*:*test_flatten_axis0*:*test_transpose_default*:*test_sum_one_input*"
FAILED_TEST_CASES="*test_averagepool_1d_default*:*test_average_2d_precomputed_same_upper*:*test_reshape_reduced_dims*:*test_maxpool_with_argmax_2d_precomputed_pads*:*test_reshape_negative_dim*:*test_maxpool_3d_default*:*test_maxpool_2d_same_upper*:*test_averagepool_2d_precomputed_same_upper*:*test_reshape_extended_dims*:*test_averagepool_2d_same_upper*:*test_gemm_broadcast*:*test_gemm_broadcast*:*test_reshape_one_dim*:*test_averagepool_2d_pads*:*test_maxpool_1d_default*:*test_maxpool_2d_same_lower*:*test_gemm_nobroadcast*:*test_maxpool_2d_precomputed_same_upper*:*test_maxpool_with_argmax_2d_precomputed_strides*:*test_averagepool_2d_precomputed_pads*:*test_reshape_reordered_dims*:*test_averagepool_2d_same_lower*:*test_averagepool_3d_default*"
EXCLUDED_TEST_CASES=${CRASHED_TEST_CASES}":"${FAILED_TEST_CASES}

# Setup glow onnxifi backend so test driver can load it
cp ${GLOW_BUILD_DIR}/lib/Onnxifi/libonnxifi-glow.so ${GLOW_DIR}/libonnxifi.so
export LD_LIBRARY_PATH=${GLOW_DIR}

# Run Onnxifi gtest
GTEST_FILTER=*-${EXCLUDED_TEST_CASES} ${GLOW_BUILD_DIR}/onnxifi_test_driver_gtests ${TESTDATA_DIR}
fi
@@ -27,26 +27,6 @@ before_install:

jobs:
include:
- env:
- TEST_NAME=ASAN
install:
- export PATH="/usr/lib/ccache:$PATH"
- mkdir build && cd build
- CC=clang-6.0 CXX=clang++-6.0 cmake -G Ninja
-DCMAKE_BUILD_TYPE=Release -DGLOW_WITH_OPENCL=OFF -DGLOW_WITH_CPU=ON
-DCMAKE_PREFIX_PATH=/usr/lib/llvm-6.0/include/
-DGLOW_USE_SANITIZER="Address;Undefined"
../

- env:
- TEST_NAME=DEBUG
install:
- mkdir build && cd build
- CC=clang-6.0 CXX=clang++-6.0 cmake -G Ninja
-DCMAKE_BUILD_TYPE=Debug -DGLOW_WITH_OPENCL=OFF -DGLOW_WITH_CPU=ON
-DCMAKE_PREFIX_PATH=/usr/lib/llvm-6.0/include/
../

- compiler: g++
env:
- TEST_NAME=COVERAGE
@@ -72,6 +52,3 @@ jobs:
- sudo apt-get install -y clang-format-6.0
script:
- CLANG_COMMAND=/usr/bin/clang-format-6.0 ./utils/format.sh check

script:
- ../.travis/run_tests.sh ${TEST_NAME}

This file was deleted.

Oops, something went wrong.

0 comments on commit 6fd7425

Please sign in to comment.