Make PYTORCH CI faster (#3687)

Summary:
By building less things.

Documentation:

[Optional Fixes #issue]
Pull Request resolved: #3687

Test Plan: Please see a detailed explanation of how to fill out the fields in the relevant sections in PULL_REQUEST.md.

Differential Revision: D18161974

Pulled By: yinghai

fbshipit-source-id: 4f4d67c06f8b083ea2557bde7259aaf581e53ae6

.circleci/build.sh

@@ -83,16 +83,15 @@ GLOW_DIR=$PWD
 cd ${GLOW_DIR}
 mkdir build && cd build
 
-if [[ "${CIRCLE_JOB}" == "PYTORCH" ]]; then
-    CMAKE_ARGS=("-DCMAKE_CXX_COMPILER=/usr/bin/clang++-7")
-    CMAKE_ARGS+=("-DCMAKE_C_COMPILER=/usr/bin/clang-7")
-else
-    CMAKE_ARGS=("-DCMAKE_CXX_COMPILER=/usr/bin/clang++-8")
+CMAKE_ARGS=()
+# PYTORCH build are directly using the sccache wrappers
+if [[ "${CIRCLE_JOB}" != "PYTORCH" ]]; then
+    CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=/usr/bin/clang++-8")
     CMAKE_ARGS+=("-DCMAKE_C_COMPILER=/usr/bin/clang-8")
+    CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER_LAUNCHER=sccache")
+    CMAKE_ARGS+=("-DCMAKE_C_COMPILER_LAUNCHER=sccache")
 fi
 
-CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER_LAUNCHER=sccache")
-CMAKE_ARGS+=("-DCMAKE_C_COMPILER_LAUNCHER=sccache")
 CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-Werror")
 CMAKE_ARGS+=("-DGLOW_WITH_CPU=ON")
 CMAKE_ARGS+=("-DGLOW_WITH_HABANA=OFF")
@@ -133,10 +132,10 @@ elif [[ "$CIRCLE_JOB" == "PYTORCH" ]]; then
     cd /tmp
     python3.6 -m virtualenv venv
     source venv/bin/activate
-    git clone https://github.com/pytorch/pytorch.git --recursive
+    git clone https://github.com/pytorch/pytorch.git --recursive --depth 1
     cd pytorch
     pip install -r requirements.txt
-    python setup.py install
+    BUILD_BINARY=OFF BUILD_TEST=0 BUILD_CAFFE2_OPS=0 python setup.py install
     cd ${GLOW_DIR}
     cd build
 elif [[ "$CIRCLE_JOB" == "OPENCL" ]]; then
@@ -149,12 +148,12 @@ else
     fi
 fi
 
-if [ "${CIRCLE_JOB}" != "COVERAGE" ] && [ "${CIRCLE_JOB}" != "CHECK_CLANG_AND_PEP8_FORMAT" ]; then
+if [ "${CIRCLE_JOB}" != "COVERAGE" ] && [ "${CIRCLE_JOB}" != "CHECK_CLANG_AND_PEP8_FORMAT" ] && [ "${CIRCLE_JOB}" != "PYTORCH" ]; then
     cmake -GNinja ${CMAKE_ARGS[*]} ../
     ninja
+fi
 
-    # Report sccache hit/miss stats
-    if hash sccache 2>/dev/null; then
-        sccache --show-stats
-    fi
+# Report sccache hit/miss stats
+if hash sccache 2>/dev/null; then
+    sccache --show-stats
 fi

.circleci/test.sh

@@ -48,9 +48,15 @@ run_and_check_resnet50_bundle() {
 
 run_pytorch_tests() {
     cd "${GLOW_SRC}/torch_glow"
+    if hash sccache 2>/dev/null; then
+      export PATH="/tmp/sccache:$PATH"
+    fi
     source /tmp/venv/bin/activate
     python "${GLOW_SRC}/torch_glow/setup.py" test --run_cmake
     cd -
+    if hash sccache 2>/dev/null; then
+      sccache --show-stats
+    fi
 }
 
 # Run unit tests and bundle tests.

torch_glow/setup.py

@@ -125,6 +125,7 @@ def _run_cmake(self):
                 '-DC10_USE_GLOG=1',
                 '-DCMAKE_BUILD_RTTI=ON',
                 '-DGLOW_BUILD_PYTORCH_INTEGRATION=ON',
+                '-DGLOW_BUILD_EXAMPLES=OFF',
                 '-DBUILD_SHARED_LIBS=OFF',
                 '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
                 '-DCMAKE_BUILD_TYPE={}'.format(