Skip to content

Merge job-spec env variables of Pytorch/Caffe2 CI jobs #16649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Merge job-spec env variables of Pytorch/Caffe2 CI jobs #16649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
561 changes: 268 additions & 293 deletions .circleci/config.yml
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions .jenkins/caffe2/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ set -ex
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"

# TODO: Migrate all centos jobs to use proper devtoolset
if [[ "$BUILD_ENVIRONMENT" == "py2-cuda9.0-cudnn7-centos7" ]]; then
if [[ "$BUILD_ENVIRONMENT" == *py2-cuda9.0-cudnn7-centos7* ]]; then
# There is a bug in pango packge on Centos7 that causes undefined
# symbols, upgrading glib2 to >=2.56.1 solves the issue. See
# https://bugs.centos.org/view.php?id=15495
Expand Down Expand Up @@ -134,7 +134,7 @@ build_args+=("BUILD_TEST=ON")
build_args+=("INSTALL_TEST=ON")
build_args+=("USE_ZSTD=ON")

if [[ $BUILD_ENVIRONMENT == py2-cuda9.0-cudnn7-ubuntu16.04 ]]; then
if [[ $BUILD_ENVIRONMENT == *py2-cuda9.0-cudnn7-ubuntu16.04* ]]; then
# removing http:// duplicate in favor of nvidia-ml.list
# which is https:// version of the same repo
sudo rm -f /etc/apt/sources.list.d/nvidia-machine-learning.list
Expand Down
2 changes: 1 addition & 1 deletion .jenkins/pytorch/build-asan.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# (This is set by default in the Docker images we build, so you don't
# need to set it yourself.

COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-build"
COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"

echo "Clang version:"
Expand Down
8 changes: 4 additions & 4 deletions .jenkins/pytorch/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# (This is set by default in the Docker images we build, so you don't
# need to set it yourself.

COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-build"
COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"

# For distributed, four environmental configs:
Expand All @@ -30,7 +30,7 @@ if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9*gcc7* ]] || [[ "$BUILD_ENVIRONMENT"
sudo mkdir -p /var/run/sshd
fi

if [[ "$BUILD_ENVIRONMENT" == "pytorch-linux-xenial-py3-clang5-asan" ]]; then
if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-py3-clang5-asan* ]]; then
exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" $*
fi

Expand Down Expand Up @@ -116,7 +116,7 @@ if [[ "$BUILD_ENVIRONMENT" == *trusty-py3.6-gcc5.4* ]]; then
fi

# Patch required to build xla
if [[ "${JOB_BASE_NAME}" == *xla* ]]; then
if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
git clone --recursive https://github.com/pytorch/xla.git
patch -p1 < xla/pytorch.patch
fi
Expand Down Expand Up @@ -176,7 +176,7 @@ if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
fi

# Test XLA build
if [[ "${JOB_BASE_NAME}" == *xla* ]]; then
if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
# TODO: Move this to Dockerfile.
# Bazel doesn't work with sccache gcc. https://github.com/bazelbuild/bazel/issues/3642
sudo add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-7 main"
Expand Down
8 changes: 4 additions & 4 deletions .jenkins/pytorch/enabled-configs.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@ pytorch-macos-10.13-cuda9.2-cudnn7-py3-build
pytorch-docker-build-test
short-perf-test-cpu
short-perf-test-gpu
py2-clang7-rocmdeb-ubuntu16.04-build
py2-clang7-rocmdeb-ubuntu16.04-test
py2-devtoolset7-rocmrpm-centos7.5-build
py2-devtoolset7-rocmrpm-centos7.5-test
py2-clang7-rocmdeb-ubuntu16.04
py2-devtoolset7-rocmrpm-centos7.5
pytorch-ppc64le-cuda9.2-cudnn7-py3-build
pytorch-ppc64le-cuda9.2-cudnn7-py3-test
pytorch-ppc64le-cuda9.1-cudnn7-py3-build
pytorch-ppc64le-cuda9.1-cudnn7-py3-test
pytorch-linux-xenial-cuda8-cudnn7-py3-NO_AVX2-test
pytorch-linux-xenial-cuda8-cudnn7-py3-NO_AVX-NO_AVX2-test
4 changes: 2 additions & 2 deletions .jenkins/pytorch/macos-build-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/bin/bash

if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-build* ]]; then
if [ -z "${BUILD_ENVIRONMENT}" ] || [[ "${BUILD_ENVIRONMENT}" == *-build* ]]; then
source "$(dirname "${BASH_SOURCE[0]}")/macos-build.sh"
fi

if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test* ]]; then
if [ -z "${BUILD_ENVIRONMENT}" ] || [[ "${BUILD_ENVIRONMENT}" == *-test* ]]; then
source "$(dirname "${BASH_SOURCE[0]}")/macos-test.sh"
fi
6 changes: 3 additions & 3 deletions .jenkins/pytorch/macos-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-build"
COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
export PATH="/usr/local/bin:$PATH"
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"

Expand All @@ -22,7 +22,7 @@ git submodule update --init --recursive
export CMAKE_PREFIX_PATH=${PYTORCH_ENV_DIR}/miniconda3/

# Build PyTorch
if [[ "${JOB_BASE_NAME}" == *cuda9.2* ]]; then
if [[ "${BUILD_ENVIRONMENT}" == *cuda9.2* ]]; then
export CUDA_VERSION=9.2
export TORCH_CUDA_ARCH_LIST=5.2
export PATH=/Developer/NVIDIA/CUDA-${CUDA_VERSION}/bin${PATH:+:${PATH}}
Expand Down Expand Up @@ -51,7 +51,7 @@ if which sccache > /dev/null; then
printf "#!/bin/sh\nexec sccache $(which clang) \$*" > "${PYTORCH_ENV_DIR}/clang"
chmod a+x "${PYTORCH_ENV_DIR}/clang"

if [[ "${JOB_BASE_NAME}" == *cuda* ]]; then
if [[ "${BUILD_ENVIRONMENT}" == *cuda* ]]; then
printf "#!/bin/sh\nexec sccache $(which nvcc) \$*" > "${PYTORCH_ENV_DIR}/nvcc"
chmod a+x "${PYTORCH_ENV_DIR}/nvcc"
export CUDA_NVCC_EXECUTABLE="${PYTORCH_ENV_DIR}/nvcc"
Expand Down
10 changes: 5 additions & 5 deletions .jenkins/pytorch/macos-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-test"
COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"

export PATH="/usr/local/bin:$PATH"
Expand All @@ -27,7 +27,7 @@ export CMAKE_PREFIX_PATH=${PYTORCH_ENV_DIR}/miniconda3/

# Test PyTorch
if [ -z "${IN_CIRCLECI}" ]; then
if [[ "${JOB_BASE_NAME}" == *cuda9.2* ]]; then
if [[ "${BUILD_ENVIRONMENT}" == *cuda9.2* ]]; then
# Eigen gives "explicit specialization of class must precede its first use" error
# when compiling with Xcode 9.1 toolchain, so we have to use Xcode 8.2 toolchain instead.
export DEVELOPER_DIR=/Library/Developer/CommandLineTools
Expand Down Expand Up @@ -108,14 +108,14 @@ test_custom_script_ops() {
}


if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
if [ -z "${BUILD_ENVIRONMENT}" ] || [[ "${BUILD_ENVIRONMENT}" == *-test ]]; then
test_python_all
test_libtorch
test_custom_script_ops
else
if [[ "${JOB_BASE_NAME}" == *-test1 ]]; then
if [[ "${BUILD_ENVIRONMENT}" == *-test1 ]]; then
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like dead code to me. @yf225 did there used to be more mac test jobs?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We used to split the test job in half for parallel testing which is what this was for. We can probably keep them around in case we are doing it in the future (which would be a pretty nice CI speed improvement)

test_python_all
elif [[ "${JOB_BASE_NAME}" == *-test2 ]]; then
elif [[ "${BUILD_ENVIRONMENT}" == *-test2 ]]; then
test_libtorch
test_custom_script_ops
fi
Expand Down
2 changes: 1 addition & 1 deletion .jenkins/pytorch/multigpu-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# (This is set by default in the Docker images we build, so you don't
# need to set it yourself.

COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-multigpu-test"
COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"

echo "Testing pytorch (distributed only)"
Expand Down
14 changes: 7 additions & 7 deletions .jenkins/pytorch/test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# (This is set by default in the Docker images we build, so you don't
# need to set it yourself.

COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-test"
COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"

echo "Testing pytorch"
Expand Down Expand Up @@ -85,9 +85,9 @@ if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
pip install -q psutil "librosa>=0.6.2" --user
fi

if [[ "${JOB_BASE_NAME}" == *-NO_AVX-* ]]; then
if [[ "${BUILD_ENVIRONMENT}" == *-NO_AVX-* ]]; then
export ATEN_CPU_CAPABILITY=default
elif [[ "${JOB_BASE_NAME}" == *-NO_AVX2-* ]]; then
elif [[ "${BUILD_ENVIRONMENT}" == *-NO_AVX2-* ]]; then
export ATEN_CPU_CAPABILITY=avx
fi

Expand Down Expand Up @@ -186,8 +186,8 @@ test_xla() {
assert_git_not_dirty
}

if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
if [[ "${JOB_BASE_NAME}" == *xla* ]]; then
if [ -z "${BUILD_ENVIRONMENT}" ] || [[ "${BUILD_ENVIRONMENT}" == *-test ]]; then
if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
test_torchvision
test_xla
else
Expand All @@ -199,10 +199,10 @@ if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
test_custom_script_ops
fi
else
if [[ "${JOB_BASE_NAME}" == *-test1 ]]; then
if [[ "${BUILD_ENVIRONMENT}" == *-test1 ]]; then
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks dead as well

test_torchvision
test_python_nn
elif [[ "${JOB_BASE_NAME}" == *-test2 ]]; then
elif [[ "${BUILD_ENVIRONMENT}" == *-test2 ]]; then
test_torchvision
test_python_all_except_nn
test_aten
Expand Down
6 changes: 3 additions & 3 deletions .jenkins/pytorch/win-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,15 +148,15 @@ test_api.exe --gtest_filter="-IntegrationTest.MNIST*"
EOL

run_tests() {
if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
if [ -z "${BUILD_ENVIRONMENT}" ] || [[ "${BUILD_ENVIRONMENT}" == *-test ]]; then
$TMP_DIR/ci_scripts/test_python_nn.bat && \
$TMP_DIR/ci_scripts/test_python_all_except_nn.bat && \
$TMP_DIR/ci_scripts/test_custom_script_ops.bat && \
$TMP_DIR/ci_scripts/test_libtorch.bat
else
if [[ "${JOB_BASE_NAME}" == *-test1 ]]; then
if [[ "${BUILD_ENVIRONMENT}" == *-test1 ]]; then
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here as well

Copy link
Member

@kostmo kostmo Feb 14, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe the Windows tests are no longer running at all after this change.

Note that the value of ${JOB_BASE_NAME} used to be pytorch-win-ws2016-cuda9-cudnn7-py3-test1 , but now ${BUILD_ENVIRONMENT} is just pytorch-win-ws2016-cuda9-cudnn7-py3 (without the -test1 suffix), so none of the conditions in the run_tests() function are matched.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Filed Issue #17101

$TMP_DIR/ci_scripts/test_python_nn.bat
elif [[ "${JOB_BASE_NAME}" == *-test2 ]]; then
elif [[ "${BUILD_ENVIRONMENT}" == *-test2 ]]; then
$TMP_DIR/ci_scripts/test_python_all_except_nn.bat && \
$TMP_DIR/ci_scripts/test_custom_script_ops.bat && \
$TMP_DIR/ci_scripts/test_libtorch.bat
Expand Down