add new build files to gitignore; test that build does not leave git …

…repo checkout dirty (pytorch#16565)

Summary:
These appear when I run
```
MACOSX_DEPLOYMENT_TARGET=10.13 CC=clang CXX=clang++ NO_CUDA=1 NO_DISTRIBUTED=1 BUILD_CAFFE2_OPS=0 DEBUG=1 python3 setup.py develop --cmake
```
Pull Request resolved: pytorch#16565

Differential Revision: D13885790

Pulled By: ezyang

fbshipit-source-id: af0e028d7fa7832a945aaee4e241ceb5418f4ec8

.circleci/config.yml

@@ -80,7 +80,7 @@ install_doc_push_script: &install_doc_push_script
     popd
     pushd docs
     rm -rf source/torchvision
-    cp -r ../vision/docs/source source/torchvision
+    cp -a ../vision/docs/source source/torchvision
 
     # Build the docs
     pip -q install -r requirements.txt || true
@@ -1242,12 +1242,13 @@ jobs:
             export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V3}
             export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V3}
 
-            git submodule sync && git submodule update -q --init
             chmod a+x .jenkins/pytorch/macos-build.sh
             unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
 
             mkdir -p /Users/distiller/pytorch-ci-env/workspace
-            cp -r /Users/distiller/project/. /Users/distiller/pytorch-ci-env/workspace
+
+            # copy with -a to preserve relative structure (e.g., symlinks), and be recursive
+            cp -a /Users/distiller/project/. /Users/distiller/pytorch-ci-env/workspace
       - persist_to_workspace:
           root: /Users/distiller/pytorch-ci-env
           paths:
@@ -1276,7 +1277,8 @@ jobs:
             set -e
             export IN_CIRCLECI=1
 
-            cp -r /Users/distiller/pytorch-ci-env/workspace/. /Users/distiller/project
+            # copy with -a to preserve relative structure (e.g., symlinks), and be recursive
+            cp -a /Users/distiller/pytorch-ci-env/workspace/. /Users/distiller/project
 
             chmod a+x .jenkins/pytorch/macos-test.sh
             unbuffer .jenkins/pytorch/macos-test.sh 2>&1 | ts

.gitignore

@@ -22,7 +22,6 @@
 aten/build/
 aten/src/ATen/Config.h
 aten/src/ATen/cuda/CUDAConfig.h
-build/
 caffe2/cpp_test/
 dist/
 docs/src/**/*
@@ -55,23 +54,33 @@ torch/csrc/nn/THNN_generic.cwrap
 torch/csrc/nn/THNN_generic.h
 torch/csrc/nn/THNN.cpp
 torch/csrc/nn/THNN.cwrap
+torch/bin/
+torch/cmake/
 torch/lib/*.a*
 torch/lib/*.dll*
 torch/lib/*.exe*
 torch/lib/*.dylib*
 torch/lib/*.h
 torch/lib/*.lib
 torch/lib/*.so*
+torch/lib/protobuf*.pc
 torch/lib/build
+torch/lib/caffe2/
 torch/lib/cmake
 torch/lib/include
 torch/lib/pkgconfig
 torch/lib/protoc
 torch/lib/tmp_install
 torch/lib/torch_shm_manager
+torch/lib/site-packages/
 torch/lib/python*
+torch/include/
 torch/share/
+torch/test/
 torch/version.py
+# Root level file used in CI to specify certain env configs.
+# E.g., see .circleci/config.yaml
+env
 
 # IPython notebook checkpoints
 .ipynb_checkpoints

.jenkins/pytorch/build-asan.sh

@@ -19,3 +19,5 @@ CC="clang" CXX="clang++" LDSHARED="clang --shared" \
   CFLAGS="-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -shared-libasan" \
   NO_CUDA=1 USE_MKLDNN=0 \
   python setup.py install
+
+assert_git_not_dirty

.jenkins/pytorch/build.sh

@@ -126,13 +126,11 @@ fi
 # only use for "python setup.py install" line
 if [[ "$BUILD_ENVIRONMENT" != *ppc64le* ]]; then
   WERROR=1 python setup.py install
-elif [[ "$BUILD_ENVIRONMENT" == *ppc64le* ]]; then
+else
   python setup.py install
 fi
 
-
-# Add the test binaries so that they won't be git clean'ed away
-git add -f build/bin
+assert_git_not_dirty
 
 # Test documentation build
 if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda8-cudnn7-py3* ]]; then
@@ -141,6 +139,7 @@ if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda8-cudnn7-py3* ]]; then
   pip install -q -r requirements.txt || true
   LC_ALL=C make html
   popd
+  assert_git_not_dirty
 fi
 
 # Test standalone c10 build
@@ -150,6 +149,7 @@ if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda8-cudnn7-py3* ]]; then
   cmake ..
   make -j
   popd
+  assert_git_not_dirty
 fi
 
 # Test no-Python build
@@ -172,6 +172,7 @@ if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
   CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" cmake "$CUSTOM_OP_TEST"
   make VERBOSE=1
   popd
+  assert_git_not_dirty
 fi
 
 # Test XLA build
@@ -217,4 +218,5 @@ if [[ "${JOB_BASE_NAME}" == *xla* ]]; then
 
   python setup.py install
   popd
+  assert_git_not_dirty
 fi

.jenkins/pytorch/common.sh

@@ -63,6 +63,20 @@ declare -f -t trap_add
 
 trap_add cleanup EXIT
 
+function assert_git_not_dirty() {
+    # TODO: we should add an option to `build_amd.py` that reverts the repo to
+    #       an unmodified state.
+    if [[ "$BUILD_ENVIRONMENT" != *rocm* ]]; then
+        git_status=$(git status --porcelain)
+        if [[ $git_status ]]; then
+            echo "Build left local git repository checkout dirty"
+            echo "git status --porcelain:"
+            echo "${git_status}"
+            exit 1
+        fi
+    fi
+}
+
 if which sccache > /dev/null; then
   # Save sccache logs to file
   sccache --stop-server || true
@@ -115,7 +129,8 @@ else
 fi
 
 if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-cuda9-cudnn7-py3 ]] || \
-   [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-trusty-py3.6-gcc7* ]]; then
+   [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-trusty-py3.6-gcc7* ]] || \
+   [[ "$BUILD_ENVIRONMENT" == *pytorch_macos* ]]; then
   BUILD_TEST_LIBTORCH=1
 else
   BUILD_TEST_LIBTORCH=0

.jenkins/pytorch/macos-build.sh

@@ -17,6 +17,7 @@ source ${PYTORCH_ENV_DIR}/miniconda3/bin/activate
 conda install -y mkl mkl-include numpy pyyaml setuptools cmake cffi ninja
 rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
 
+git submodule sync --recursive
 git submodule update --init --recursive
 export CMAKE_PREFIX_PATH=${PYTORCH_ENV_DIR}/miniconda3/
 
@@ -65,6 +66,8 @@ export IMAGE_COMMIT_TAG=${BUILD_ENVIRONMENT}-${IMAGE_COMMIT_ID}
 
 python setup.py install
 
+assert_git_not_dirty
+
 # Upload torch binaries when the build job is finished
 if [ -z "${IN_CIRCLECI}" ]; then
   7z a ${IMAGE_COMMIT_TAG}.7z ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*

.jenkins/pytorch/macos-test.sh

@@ -16,11 +16,12 @@ fi
 export PATH="${PYTORCH_ENV_DIR}/miniconda3/bin:$PATH"
 source ${PYTORCH_ENV_DIR}/miniconda3/bin/activate
 conda install -y mkl mkl-include numpy pyyaml setuptools cmake cffi ninja six
-pip install hypothesis librosa>=0.6.2 psutil
+pip install -q hypothesis "librosa>=0.6.2" psutil
 if [ -z "${IN_CIRCLECI}" ]; then
   rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
 fi
 
+git submodule sync --recursive
 git submodule update --init --recursive
 export CMAKE_PREFIX_PATH=${PYTORCH_ENV_DIR}/miniconda3/
 
@@ -52,31 +53,38 @@ fi
 test_python_all() {
   echo "Ninja version: $(ninja --version)"
   python test/run_test.py --verbose
+  assert_git_not_dirty
 }
 
-test_cpp_api() {
+test_libtorch() {
   # C++ API
 
-  # NB: Install outside of source directory (at the same level as the root
-  # pytorch folder) so that it doesn't get cleaned away prior to docker push.
-  # But still clean it before we perform our own build.
-  #
-  CPP_BUILD="$PWD/../cpp-build"
-  rm -rf $CPP_BUILD
-  mkdir -p $CPP_BUILD/caffe2
-
-  BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
-  pushd $CPP_BUILD/caffe2
-  VERBOSE=1 DEBUG=1 python $BUILD_LIBTORCH_PY
-  popd
+  if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
+    # NB: Install outside of source directory (at the same level as the root
+    # pytorch folder) so that it doesn't get cleaned away prior to docker push.
+    # But still clean it before we perform our own build.
+
+    echo "Testing libtorch"
+
+    CPP_BUILD="$PWD/../cpp-build"
+    rm -rf $CPP_BUILD
+    mkdir -p $CPP_BUILD/caffe2
 
-  python tools/download_mnist.py --quiet -d test/cpp/api/mnist
+    BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
+    pushd $CPP_BUILD/caffe2
+    VERBOSE=1 DEBUG=1 python $BUILD_LIBTORCH_PY
+    popd
 
-  # Unfortunately it seems like the test can't load from miniconda3
-  # without these paths being set
-  export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/miniconda3/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/miniconda3/lib"
-  "$CPP_BUILD"/caffe2/bin/test_api
+    python tools/download_mnist.py --quiet -d test/cpp/api/mnist
+
+    # Unfortunately it seems like the test can't load from miniconda3
+    # without these paths being set
+    export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/miniconda3/lib"
+    export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/miniconda3/lib"
+    TORCH_CPP_TEST_MNIST_PATH="test/cpp/api/mnist" "$CPP_BUILD"/caffe2/bin/test_api
+
+    assert_git_not_dirty
+  fi
 }
 
 test_custom_script_ops() {
@@ -96,18 +104,19 @@ test_custom_script_ops() {
   # Run tests C++-side and load the exported script module.
   build/test_custom_ops ./model.pt
   popd
+  assert_git_not_dirty
 }
 
 
 if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
   test_python_all
-  test_cpp_api
+  test_libtorch
   test_custom_script_ops
 else
   if [[ "${JOB_BASE_NAME}" == *-test1 ]]; then
     test_python_all
   elif [[ "${JOB_BASE_NAME}" == *-test2 ]]; then
-    test_cpp_api
+    test_libtorch
     test_custom_script_ops
   fi
 fi

.jenkins/pytorch/multigpu-test.sh

@@ -26,3 +26,4 @@ if [ -n "${IN_CIRCLECI}" ]; then
 fi
 
 time python test/run_test.py --verbose -i distributed
+assert_git_not_dirty

.jenkins/pytorch/perf_test/test_cpu_speed_mnist.sh

@@ -12,7 +12,7 @@ test_cpu_speed_mnist () {
 
   cd examples/mnist
 
-  pip install -r requirements.txt
+  pip install -q -r requirements.txt
 
   # Download data
   python main.py --epochs 0

.jenkins/pytorch/perf_test/test_gpu_speed_mnist.sh

@@ -12,7 +12,7 @@ test_gpu_speed_mnist () {
 
   cd examples/mnist
 
-  pip install -r requirements.txt
+  pip install -q -r requirements.txt
 
   # Download data
   python main.py --epochs 0

.jenkins/pytorch/short-perf-test-cpu.sh

@@ -7,7 +7,7 @@ cd .jenkins/pytorch/perf_test
 
 echo "Running CPU perf test for PyTorch..."
 
-pip install awscli
+pip install -q awscli
 
 # Set multipart_threshold to be sufficiently high, so that `aws s3 cp` is not a multipart read
 # More info at https://github.com/aws/aws-cli/issues/2321

.jenkins/pytorch/short-perf-test-gpu.sh

@@ -7,7 +7,7 @@ pushd .jenkins/pytorch/perf_test
 
 echo "Running GPU perf test for PyTorch..."
 
-pip install awscli
+pip install -q awscli
 
 # Set multipart_threshold to be sufficiently high, so that `aws s3 cp` is not a multipart read
 # More info at https://github.com/aws/aws-cli/issues/2321

.jenkins/pytorch/test.sh

@@ -82,7 +82,7 @@ if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
   export PYTORCH_TEST_WITH_ROCM=1
   # ROCm CI is using Caffe2 docker images, which doesn't have several packages
   # needed in testing. We install them here.
-  pip install -q psutil librosa>=0.6.2 --user
+  pip install -q psutil "librosa>=0.6.2" --user
 fi
 
 if [[ "${JOB_BASE_NAME}" == *-NO_AVX-* ]]; then
@@ -93,10 +93,12 @@ fi
 
 test_python_nn() {
   time python test/run_test.py --include nn --verbose
+  assert_git_not_dirty
 }
 
 test_python_all_except_nn() {
   time python test/run_test.py --exclude nn --verbose
+  assert_git_not_dirty
 }
 
 test_aten() {
@@ -120,6 +122,7 @@ test_aten() {
 
     ls build/bin
     aten/tools/run_tests.sh build/bin
+    assert_git_not_dirty
   fi
 }
 
@@ -139,19 +142,21 @@ test_torchvision() {
   #time python setup.py install
   pip install -q --user .
   popd
+  rm -rf vision
 }
 
 test_libtorch() {
   if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
-     echo "Testing libtorch"
-     CPP_BUILD="$PWD/../cpp-build"
-     if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
-       "$CPP_BUILD"/caffe2/bin/test_jit
-     else
-       "$CPP_BUILD"/caffe2/bin/test_jit "[cpu]"
-     fi
-     python tools/download_mnist.py --quiet -d mnist
-     OMP_NUM_THREADS=2 "$CPP_BUILD"/caffe2/bin/test_api
+    echo "Testing libtorch"
+    CPP_BUILD="$PWD/../cpp-build"
+    if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
+      "$CPP_BUILD"/caffe2/bin/test_jit
+    else
+      "$CPP_BUILD"/caffe2/bin/test_jit "[cpu]"
+    fi
+    python tools/download_mnist.py --quiet -d test/cpp/api/mnist
+    OMP_NUM_THREADS=2 TORCH_CPP_TEST_MNIST_PATH="test/cpp/api/mnist" "$CPP_BUILD"/caffe2/bin/test_api
+    assert_git_not_dirty
   fi
 }
 
@@ -160,13 +165,14 @@ test_custom_script_ops() {
     echo "Testing custom script operators"
     CUSTOM_OP_BUILD="$PWD/../custom-op-build"
     pushd test/custom_operator
-    cp -r "$CUSTOM_OP_BUILD" build
+    cp -a "$CUSTOM_OP_BUILD" build
     # Run tests Python-side and export a script module.
     python test_custom_ops.py -v
     python model.py --export-script-module=model.pt
     # Run tests C++-side and load the exported script module.
     build/test_custom_ops ./model.pt
     popd
+    assert_git_not_dirty
   fi
 }
 
@@ -177,6 +183,7 @@ test_xla() {
   python test/test_operations.py
   python test/test_train_mnist.py --tidy
   popd
+  assert_git_not_dirty
 }
 
 if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then