Skip to content
Permalink
Browse files

Make PYTORCH CI faster (#3687)

Summary:
By building less things.

Documentation:

[Optional Fixes #issue]
Pull Request resolved: #3687

Test Plan: Please see a detailed explanation of how to fill out the fields in the relevant sections in PULL_REQUEST.md.

Differential Revision: D18161974

Pulled By: yinghai

fbshipit-source-id: 4f4d67c06f8b083ea2557bde7259aaf581e53ae6
  • Loading branch information...
yinghai authored and facebook-github-bot committed Oct 26, 2019
1 parent e00a8b4 commit 59c33b9023f8cdd99f30fe7abd8080809753fe7b
Showing with 20 additions and 14 deletions.
  1. +13 −14 .circleci/build.sh
  2. +6 −0 .circleci/test.sh
  3. +1 −0 torch_glow/setup.py
@@ -83,16 +83,15 @@ GLOW_DIR=$PWD
cd ${GLOW_DIR}
mkdir build && cd build

if [[ "${CIRCLE_JOB}" == "PYTORCH" ]]; then
CMAKE_ARGS=("-DCMAKE_CXX_COMPILER=/usr/bin/clang++-7")
CMAKE_ARGS+=("-DCMAKE_C_COMPILER=/usr/bin/clang-7")
else
CMAKE_ARGS=("-DCMAKE_CXX_COMPILER=/usr/bin/clang++-8")
CMAKE_ARGS=()
# PYTORCH build are directly using the sccache wrappers
if [[ "${CIRCLE_JOB}" != "PYTORCH" ]]; then
CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=/usr/bin/clang++-8")
CMAKE_ARGS+=("-DCMAKE_C_COMPILER=/usr/bin/clang-8")
CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER_LAUNCHER=sccache")
CMAKE_ARGS+=("-DCMAKE_C_COMPILER_LAUNCHER=sccache")
fi

CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER_LAUNCHER=sccache")
CMAKE_ARGS+=("-DCMAKE_C_COMPILER_LAUNCHER=sccache")
CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-Werror")
CMAKE_ARGS+=("-DGLOW_WITH_CPU=ON")
CMAKE_ARGS+=("-DGLOW_WITH_HABANA=OFF")
@@ -133,10 +132,10 @@ elif [[ "$CIRCLE_JOB" == "PYTORCH" ]]; then
cd /tmp
python3.6 -m virtualenv venv
source venv/bin/activate
git clone https://github.com/pytorch/pytorch.git --recursive
git clone https://github.com/pytorch/pytorch.git --recursive --depth 1
cd pytorch
pip install -r requirements.txt
python setup.py install
BUILD_BINARY=OFF BUILD_TEST=0 BUILD_CAFFE2_OPS=0 python setup.py install
cd ${GLOW_DIR}
cd build
elif [[ "$CIRCLE_JOB" == "OPENCL" ]]; then
@@ -149,12 +148,12 @@ else
fi
fi

if [ "${CIRCLE_JOB}" != "COVERAGE" ] && [ "${CIRCLE_JOB}" != "CHECK_CLANG_AND_PEP8_FORMAT" ]; then
if [ "${CIRCLE_JOB}" != "COVERAGE" ] && [ "${CIRCLE_JOB}" != "CHECK_CLANG_AND_PEP8_FORMAT" ] && [ "${CIRCLE_JOB}" != "PYTORCH" ]; then
cmake -GNinja ${CMAKE_ARGS[*]} ../
ninja
fi

# Report sccache hit/miss stats
if hash sccache 2>/dev/null; then
sccache --show-stats
fi
# Report sccache hit/miss stats
if hash sccache 2>/dev/null; then
sccache --show-stats
fi
@@ -48,9 +48,15 @@ run_and_check_resnet50_bundle() {

run_pytorch_tests() {
cd "${GLOW_SRC}/torch_glow"
if hash sccache 2>/dev/null; then
export PATH="/tmp/sccache:$PATH"
fi
source /tmp/venv/bin/activate
python "${GLOW_SRC}/torch_glow/setup.py" test --run_cmake
cd -
if hash sccache 2>/dev/null; then
sccache --show-stats
fi
}

# Run unit tests and bundle tests.
@@ -125,6 +125,7 @@ def _run_cmake(self):
'-DC10_USE_GLOG=1',
'-DCMAKE_BUILD_RTTI=ON',
'-DGLOW_BUILD_PYTORCH_INTEGRATION=ON',
'-DGLOW_BUILD_EXAMPLES=OFF',
'-DBUILD_SHARED_LIBS=OFF',
'-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
'-DCMAKE_BUILD_TYPE={}'.format(

0 comments on commit 59c33b9

Please sign in to comment.
You can’t perform that action at this time.