Skip to content
Permalink
Browse files

Make CI for glow-torch (#3164)

Summary:
Documentation:

[Optional Fixes #issue]
Pull Request resolved: #3164

Test Plan: Please see a detailed explanation of how to fill out the fields in the relevant sections in PULL_REQUEST.md.

Differential Revision: D16155330

Pulled By: zrphercule

fbshipit-source-id: ddb53ece8a7064fb00ccbde4ad73e5d942f1e243
  • Loading branch information...
zrphercule authored and facebook-github-bot committed Jul 11, 2019
1 parent db6804c commit 05c925ecd21a89080470b4355ebf6ee0c10bed74
Showing with 74 additions and 12 deletions.
  1. +54 −10 .circleci/build.sh
  2. +7 −1 .circleci/config.yml
  3. +10 −0 .circleci/test.sh
  4. +1 −1 torch_glow/src/CMakeLists.txt
  5. +2 −0 torch_glow/tests/nodes/conv2d_test.py
@@ -5,6 +5,20 @@
set -ex

export MAX_JOBS=8
if [ "${CIRCLE_JOB}" != "COVERAGE" ]; then
if hash sccache 2>/dev/null; then
SCCACHE_BIN_DIR="/tmp/sccache"
mkdir -p "$SCCACHE_BIN_DIR"
for compiler in cc c++ gcc g++ x86_64-linux-gnu-gcc; do
(
echo "#!/bin/sh"
echo "exec $(which sccache) $(which $compiler) \"\$@\""
) > "$SCCACHE_BIN_DIR/$compiler"
chmod +x "$SCCACHE_BIN_DIR/$compiler"
done
export PATH="$SCCACHE_BIN_DIR:$PATH"
fi
fi

install_pocl() {
sudo apt-get install -y ocl-icd-opencl-dev clinfo libhwloc-dev opencl-headers
@@ -24,7 +38,23 @@ install_pocl() {
cd ../
}

if [ "${CIRCLE_JOB}" != "CHECK_CLANG_FORMAT" ]; then
if [ "${CIRCLE_JOB}" == "CHECK_CLANG_FORMAT" ]; then
sudo -E apt-add-repository -y "ppa:ubuntu-toolchain-r/test"
curl -sSL "https://build.travis-ci.org/files/gpg/llvm-toolchain-trusty-7.asc" | sudo -E apt-key add -
echo "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-7 main" | sudo tee -a /etc/apt/sources.list >/dev/null
sudo apt-get update
elif [ "${CIRCLE_JOB}" == "PYTORCH" ]; then
# Install Glow dependencies
sudo apt-get update
sudo apt-get install -y llvm-7
# Redirect clang
sudo ln -s /usr/bin/clang-7 /usr/bin/clang
sudo ln -s /usr/bin/clang++-7 /usr/bin/clang++
sudo ln -s /usr/bin/llvm-symbolizer-7 /usr/bin/llvm-symbolizer
sudo ln -s /usr/bin/llvm-config-7 /usr/bin/llvm-config-7.0

sudo apt-get install -y libpng-dev libgoogle-glog-dev
else
# Install Glow dependencies
sudo apt-get update

@@ -35,11 +65,6 @@ if [ "${CIRCLE_JOB}" != "CHECK_CLANG_FORMAT" ]; then
sudo ln -s /usr/bin/llvm-config-8 /usr/bin/llvm-config-8.0

sudo apt-get install -y libpng-dev libgoogle-glog-dev
else
sudo -E apt-add-repository -y "ppa:ubuntu-toolchain-r/test"
curl -sSL "https://build.travis-ci.org/files/gpg/llvm-toolchain-trusty-7.asc" | sudo -E apt-key add -
echo "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-7 main" | sudo tee -a /etc/apt/sources.list >/dev/null
sudo apt-get update
fi

# Install ninja and (newest version of) cmake through pip
@@ -50,8 +75,15 @@ hash cmake ninja
GLOW_DIR=$PWD
cd ${GLOW_DIR}
mkdir build && cd build
CMAKE_ARGS=("-DCMAKE_CXX_COMPILER=/usr/bin/clang++-8")
CMAKE_ARGS+=("-DCMAKE_C_COMPILER=/usr/bin/clang-8")

if [[ "${CIRCLE_JOB}" == "PYTORCH" ]]; then
CMAKE_ARGS=("-DCMAKE_CXX_COMPILER=/usr/bin/clang++-7")
CMAKE_ARGS+=("-DCMAKE_C_COMPILER=/usr/bin/clang-7")
else
CMAKE_ARGS=("-DCMAKE_CXX_COMPILER=/usr/bin/clang++-8")
CMAKE_ARGS+=("-DCMAKE_C_COMPILER=/usr/bin/clang-8")
fi

CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER_LAUNCHER=sccache")
CMAKE_ARGS+=("-DCMAKE_C_COMPILER_LAUNCHER=sccache")
CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-Werror")
@@ -78,9 +110,9 @@ elif [[ "$CIRCLE_JOB" == "RELEASE_WITH_EXPENSIVE_TESTS" ]]; then
CMAKE_ARGS+=("-DGLOW_WITH_BUNDLES=ON")
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
elif [[ "$CIRCLE_JOB" == "COVERAGE" ]]; then
sudo apt-get install wget
sudo apt-get install wget
sudo apt-get install -y lcov
sudo pip install awscli --upgrade
sudo pip install awscli --upgrade
../utils/install_protobuf.sh
CC=gcc-5 CXX=g++-5 cmake -G Ninja \
-DCMAKE_BUILD_TYPE=Debug -DGLOW_WITH_OPENCL=OFF -DGLOW_WITH_CPU=ON \
@@ -89,6 +121,18 @@ elif [[ "$CIRCLE_JOB" == "COVERAGE" ]]; then
../
elif [[ "$CIRCLE_JOB" == "CHECK_CLANG_FORMAT" ]]; then
sudo apt-get install -y clang-format-7
elif [[ "$CIRCLE_JOB" == "PYTORCH" ]]; then
# Build PyTorch
cd /tmp
python3.6 -m virtualenv venv
source venv/bin/activate
git clone https://github.com/pytorch/pytorch.git --recursive
cd pytorch
git checkout 7fcfed19e7c4805405f3bec311fc056803ca7afb
pip install -r requirements.txt
python setup.py install
cd ${GLOW_DIR}
cd build
elif [[ "$CIRCLE_JOB" == "OPENCL" ]]; then
install_pocl
CMAKE_ARGS+=("-DGLOW_WITH_OPENCL=ON")
@@ -76,7 +76,7 @@ jobs:
OPENCL:
environment:
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang8-ubuntu16.04:283"
<<: *linux_default
<<: *linux_default
ASAN:
environment:
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang8-ubuntu16.04:283"
@@ -97,6 +97,11 @@ jobs:
environment:
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang8-ubuntu16.04:283"
<<: *linux_default
PYTORCH:
resource_class: 2xlarge+
environment:
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-clang7-ubuntu16.04:287"
<<: *linux_default
CHECK_CLANG_FORMAT:
environment:
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang6.0-ubuntu16.04:230"
@@ -112,4 +117,5 @@ workflows:
- SHARED
- RELEASE_WITH_EXPENSIVE_TESTS
- COVERAGE
- PYTORCH
- CHECK_CLANG_FORMAT
@@ -46,6 +46,13 @@ run_and_check_resnet50_bundle() {
done
}

run_pytorch_tests() {
cd "${GLOW_SRC}/torch_glow"
source /tmp/venv/bin/activate
python "${GLOW_SRC}/torch_glow/setup.py" test --run_cmake
cd -
}

# Run unit tests and bundle tests.
cd "${GLOW_BUILD_DIR}"
case ${CIRCLE_JOB} in
@@ -76,6 +83,9 @@ case ${CIRCLE_JOB} in
cd build
../.circleci/run_coverage.sh
;;
PYTORCH)
run_pytorch_tests
;;
CHECK_CLANG_FORMAT)
cd "${GLOW_SRC}"
sudo ln -s /usr/bin/clang-format-7 /usr/bin/clang-format
@@ -28,7 +28,7 @@ target_link_libraries(_torch_glow
Support
Backends
pybind11)

target_include_directories(_torch_glow PUBLIC
${PYTORCH_DIR}/include
)
@@ -4,6 +4,7 @@
from collections import namedtuple

from tests.utils import jitVsGlow
import pytest

# Basic test of the PyTorch conv2d Node on Glow.
def test_conv2d_basic():
@@ -32,6 +33,7 @@ def conv2d_with_bias(inputs, filters, bias):

# Test of the PyTorch conv2d Node sweeping through various parameters of the
# Node to test that they work correctly.
@pytest.mark.skip(reason="not ready")
def test_conv2d_param_sweep():
hwOpts = [3, 4]
padOpts = [0, 1]

0 comments on commit 05c925e

Please sign in to comment.
You can’t perform that action at this time.