Update jenkins caffe2 scripts for ROCm circleci images. (pytorch#39908)

Summary:
Remove work-around to install conda locally for older ROCm jenkins images.
Remove use of sudo to install pip packages.
Install missing packages for caffe2 test.sh needs on ROCm.

CC ezyang xw285cornell sunway513
Pull Request resolved: pytorch#39908

Differential Revision: D22044404

Pulled By: ezyang

fbshipit-source-id: da6b5a45dcf68432339ad6f1c4af2d8a96df73f1

.jenkins/caffe2/common.sh

@@ -6,69 +6,6 @@ TEST_DIR="$ROOT_DIR/caffe2_tests"
 gtest_reports_dir="${TEST_DIR}/cpp"
 pytest_reports_dir="${TEST_DIR}/python"
 
-# This is needed to work around ROCm using old docker images until
-# the transition to new images is complete.
-# TODO: Remove once ROCm CI is using new images.
-if [[ $BUILD_ENVIRONMENT == py3.6-devtoolset7-rocmrpm-centos* ]]; then
-  # This file is sourced multiple times, only install conda the first time.
-  # We must install conda where we have write access.
-  CONDA_DIR="$ROOT_DIR/conda"
-  if [[ ! -d $CONDA_DIR ]]; then
-    ANACONDA_PYTHON_VERSION=3.6
-    BASE_URL="https://repo.anaconda.com/miniconda"
-    CONDA_FILE="Miniconda3-latest-Linux-x86_64.sh"
-    mkdir $CONDA_DIR
-    pushd /tmp
-    wget -q "${BASE_URL}/${CONDA_FILE}"
-    chmod +x "${CONDA_FILE}"
-    ./"${CONDA_FILE}" -b -f -p "$CONDA_DIR"
-    popd
-    export PATH="$CONDA_DIR/bin:$PATH"
-    # Ensure we run conda in a directory that jenkins has write access to
-    pushd $CONDA_DIR
-    # Track latest conda update
-    conda update -n base conda
-    # Install correct Python version
-    conda install python="$ANACONDA_PYTHON_VERSION"
-
-    conda_install() {
-      # Ensure that the install command don't upgrade/downgrade Python
-      # This should be called as
-      #   conda_install pkg1 pkg2 ... [-c channel]
-      conda install -q -y python="$ANACONDA_PYTHON_VERSION" $*
-    }
-
-    # Install PyTorch conda deps, as per https://github.com/pytorch/pytorch README
-    conda_install numpy pyyaml mkl mkl-include setuptools cffi typing future six
-
-    # TODO: This isn't working atm
-    conda_install nnpack -c killeent
-
-    # Install some other packages
-
-    # Need networkx 2.0 because bellmand_ford was moved in 2.1 . Scikit-image by
-    # defaults installs the most recent networkx version, so we install this lower
-    # version explicitly before scikit-image pulls it in as a dependency
-    pip install networkx==2.0
-
-    # TODO: Why is scipy pinned
-    # numba & llvmlite is pinned because of https://github.com/numba/numba/issues/4368
-    # scikit-learn is pinned because of
-    # https://github.com/scikit-learn/scikit-learn/issues/14485 (affects gcc 5.5
-    # only)
-    pip install --progress-bar off pytest scipy==1.1.0 scikit-learn==0.20.3 scikit-image librosa>=0.6.2 psutil numba==0.46.0 llvmlite==0.30.0
-
-    # click - onnx
-    # hypothesis - tests
-    # jupyter - for tutorials
-    pip install --progress-bar off click hypothesis jupyter protobuf tabulate virtualenv mock typing-extensions
-
-    popd
-  else
-    export PATH="$CONDA_DIR/bin:$PATH"
-  fi
-fi
-
 # Figure out which Python to use
 PYTHON="$(which python)"
 if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then

.jenkins/caffe2/test.sh

@@ -11,6 +11,15 @@ if [[ "${BUILD_ENVIRONMENT}" == *-rocm* ]]; then
   # temporary to locate some kernel issues on the CI nodes
   export HSAKMT_DEBUG_LEVEL=4
 fi
+# These additional packages are needed for circleci ROCm builds.
+if [[ $BUILD_ENVIRONMENT == pytorch-linux-xenial-rocm* ]]; then
+    # Need networkx 2.0 because bellmand_ford was moved in 2.1 . Scikit-image by
+    # defaults installs the most recent networkx version, so we install this lower
+    # version explicitly before scikit-image pulls it in as a dependency
+    pip install networkx==2.0
+    # click - onnx
+    pip install --progress-bar off click protobuf tabulate virtualenv mock typing-extensions
+fi
 
 # Find where cpp tests and Caffe2 itself are installed
 if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
@@ -71,16 +80,24 @@ if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
   exit 0
 fi
 
+# If pip is installed as root, we must use sudo.
+# CircleCI docker images could install conda as jenkins user, or use the OS's python package.
+PIP=$(which pip)
+PIP_USER=$(stat --format '%U' $PIP)
+if [[ "$PIP_USER" = root ]]; then
+  MAYBE_SUDO=sudo
+fi
+
 # if [[ "$BUILD_ENVIRONMENT" == *ubuntu14.04* ]]; then
   # Hotfix, use hypothesis 3.44.6 on Ubuntu 14.04
   # See comments on
   # https://github.com/HypothesisWorks/hypothesis-python/commit/eadd62e467d6cee6216e71b391951ec25b4f5830
-  sudo pip -q uninstall -y hypothesis
+  $MAYBE_SUDO pip -q uninstall -y hypothesis
   # "pip install hypothesis==3.44.6" from official server is unreliable on
   # CircleCI, so we host a copy on S3 instead
-  sudo pip -q install attrs==18.1.0 -f https://s3.amazonaws.com/ossci-linux/wheels/attrs-18.1.0-py2.py3-none-any.whl
-  sudo pip -q install coverage==4.5.1 -f https://s3.amazonaws.com/ossci-linux/wheels/coverage-4.5.1-cp36-cp36m-macosx_10_12_x86_64.whl
-  sudo pip -q install hypothesis==3.44.6 -f https://s3.amazonaws.com/ossci-linux/wheels/hypothesis-3.44.6-py3-none-any.whl
+  $MAYBE_SUDO pip -q install attrs==18.1.0 -f https://s3.amazonaws.com/ossci-linux/wheels/attrs-18.1.0-py2.py3-none-any.whl
+  $MAYBE_SUDO pip -q install coverage==4.5.1 -f https://s3.amazonaws.com/ossci-linux/wheels/coverage-4.5.1-cp36-cp36m-macosx_10_12_x86_64.whl
+  $MAYBE_SUDO pip -q install hypothesis==3.44.6 -f https://s3.amazonaws.com/ossci-linux/wheels/hypothesis-3.44.6-py3-none-any.whl
 # else
 #   pip install --user --no-cache-dir hypothesis==3.59.0
 # fi