Skip to content

Commit

Permalink
[Enhancement] Using docker to skip CUDA installation in CI (#882)
Browse files Browse the repository at this point in the history 
* Use docker to skip CUDA installation in CI

* Update .github/workflows/build.yml

Co-authored-by: Jerry Jiarui XU <xvjiarui0826@gmail.com>

* Remove Install system dependencies

* Add Install system dependencies for cuda usage

* Add Install system dependencies and requirements

* fix requirements error

Co-authored-by: Jerry Jiarui XU <xvjiarui0826@gmail.com>
  • Loading branch information
@Junjun2016 @xvjiarui
Junjun2016 and xvjiarui committed Sep 27, 2021
1 parent 4003b8f commit 36eb2d8
Showing 1 changed file with 131 additions and 70 deletions.
201 changes: 131 additions & 70 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
name: build

on: [ push, pull_request ]
on: [push, pull_request]

jobs:
lint:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.7
uses: actions/setup-python@v1
uses: actions/setup-python@v2
with:
python-version: 3.7
- name: Install pre-commit hook
Expand All @@ -17,25 +17,36 @@ jobs:
pre-commit install
- name: Linting
run: pre-commit run --all-files
- name: Check docstring
- name: Check docstring coverage
run: |
pip install interrogate
interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --exclude mmseg/ops --ignore-regex "__repr__" --fail-under 80 mmseg
build_cpu:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
torch: [1.3.0, 1.5.0, 1.7.0, 1.9.0]
torch: [1.3.1, 1.5.1, 1.6.0, 1.7.0, 1.8.0, 1.9.0]
include:
- torch: 1.3.0
torchvision: 0.4.1
- torch: 1.5.0
torchvision: 0.6.0
- torch: 1.3.1
torchvision: 0.4.2
mmcv: "latest+torch1.3.0+cpu"
- torch: 1.5.1
torchvision: 0.6.1
mmcv: "latest+torch1.5.0+cpu"
- torch: 1.6.0
torchvision: 0.7.0
mmcv: "latest+torch1.6.0+cpu"
- torch: 1.7.0
torchvision: 0.8.1
mmcv: "latest+torch1.7.0+cpu"
- torch: 1.8.0
torchvision: 0.9.0
mmcv: "latest+torch1.8.0+cpu"
- torch: 1.9.0
torchvision: 0.10.0
mmcv: "latest+torch1.9.0+cpu"
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
Expand All @@ -45,15 +56,17 @@ jobs:
- name: Upgrade pip
run: pip install pip --upgrade
- name: Install Pillow
if: ${{matrix.torchvision == '0.4.1'}}
run: pip install Pillow==6.2.2
if: ${{matrix.torchvision == '0.4.2'}}
- name: Install PyTorch
run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
- name: Install mmseg dependencies
- name: Install MMCV
run: |
pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${{matrix.torch}}/index.html
pip install -r requirements.txt
python -c 'import mmcv; print(mmcv.__version__)'
- name: Install unittest dependencies
run: |
pip install -r requirements.txt
- name: Build and install
run: rm -rf .eggs && pip install -e .
- name: Run unittests and generate coverage report
Expand All @@ -62,98 +75,146 @@ jobs:
coverage xml
coverage report -m
build_cuda:
build_cuda101:
runs-on: ubuntu-18.04
env:
UBUNTU_VERSION: ubuntu1804
container:
image: pytorch/pytorch:1.6.0-cuda10.1-cudnn7-devel

strategy:
matrix:
python-version: [3.7]
torch: [1.5.0+cu101, 1.7.0+cu101, 1.8.0+cu101, 1.9.0+cu102]
torch:
[
1.3.1,
1.5.1+cu101,
1.6.0+cu101,
1.7.0+cu101,
1.8.0+cu101
]
include:
- torch: 1.5.0+cu101
torch_version: torch1.5.0
torchvision: 0.6.0+cu101
CUDA: 10.1.105-1
CUDA_SHORT: 10-1
- torch: 1.3.1
torch_version: torch1.3.1
torchvision: 0.4.2
mmcv_link: "torch1.3.0"
- torch: 1.5.1+cu101
torch_version: torch1.5.1
torchvision: 0.6.1+cu101
mmcv_link: "torch1.5.0"
- torch: 1.6.0+cu101
torch_version: torch1.6.0
torchvision: 0.7.0+cu101
mmcv_link: "torch1.6.0"
- torch: 1.7.0+cu101
torch_version: torch1.7.0
torchvision: 0.8.1+cu101
CUDA: 10.1.105-1
CUDA_SHORT: 10-1
mmcv_link: "torch1.7.0"
- torch: 1.8.0+cu101
torch_version: torch1.8.0
torchvision: 0.9.0+cu101
CUDA: 10.1.105-1
CUDA_SHORT: 10-1
mmcv_link: "torch1.8.0"

steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install system dependencies
run: |
apt-get update && apt-get install -y ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 python${{matrix.python-version}}-dev
apt-get clean
rm -rf /var/lib/apt/lists/*
- name: Install Pillow
run: python -m pip install Pillow==6.2.2
if: ${{matrix.torchvision < 0.5}}
- name: Install PyTorch
run: python -m pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
- name: Install dependencies for compiling onnx when python=3.9
run: python -m pip install protobuf && apt-get install libprotobuf-dev protobuf-compiler
if: ${{matrix.python-version == '3.9'}}
- name: Install mmseg dependencies
run: |
python -V
python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu101/${{matrix.mmcv_link}}/index.html
python -m pip install -r requirements.txt
python -c 'import mmcv; print(mmcv.__version__)'
- name: Build and install
run: |
rm -rf .eggs
python setup.py check -m -s
TORCH_CUDA_ARCH_LIST=7.0 pip install .
- name: Run unittests and generate coverage report
run: |
coverage run --branch --source mmseg -m pytest tests/
coverage xml
coverage report -m
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v1.0.10
with:
file: ./coverage.xml
flags: unittests
env_vars: OS,PYTHON
name: codecov-umbrella
fail_ci_if_error: false

build_cuda102:
runs-on: ubuntu-18.04
container:
image: pytorch/pytorch:1.9.0-cuda10.2-cudnn7-devel

strategy:
matrix:
python-version: [3.6, 3.7, 3.8, 3.9-dev]
torch: [1.9.0+cu102]
include:
- torch: 1.9.0+cu102
torch_version: torch1.9.0
torchvision: 0.10.0+cu102
CUDA: 10.2.89-1
CUDA_SHORT: 10-2
- python-version: 3.6
torch: 1.8.0+cu101
torch_version: torch1.8.0
torchvision: 0.9.0+cu101
CUDA: 10.1.105-1
CUDA_SHORT: 10-1
- python-version: 3.8
torch: 1.8.0+cu101
torch_version: torch1.8.0
torchvision: 0.9.0+cu101
CUDA: 10.1.105-1
CUDA_SHORT: 10-1
- python-version: 3.9
torch: 1.8.0+cu101
torch_version: torch1.8.0
torchvision: 0.9.0+cu101
CUDA: 10.1.105-1
CUDA_SHORT: 10-1
mmcv_link: "torch1.9.0"

steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: pip install pip --upgrade
- name: Install CUDA
- name: Install python-dev
run: apt-get update && apt-get install -y python${{matrix.python-version}}-dev
if: ${{matrix.python-version != '3.9-dev'}}
- name: Install system dependencies
run: |
export INSTALLER=cuda-repo-${UBUNTU_VERSION}_${{matrix.CUDA}}_amd64.deb
wget http://developer.download.nvidia.com/compute/cuda/repos/${UBUNTU_VERSION}/x86_64/${INSTALLER}
sudo dpkg -i ${INSTALLER}
wget https://developer.download.nvidia.com/compute/cuda/repos/${UBUNTU_VERSION}/x86_64/7fa2af80.pub
sudo apt-key add 7fa2af80.pub
sudo apt update -qq
sudo apt install -y cuda-${{matrix.CUDA_SHORT}} cuda-cufft-dev-${{matrix.CUDA_SHORT}}
sudo apt clean
export CUDA_HOME=/usr/local/cuda-${{matrix.CUDA_SHORT}}
export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${CUDA_HOME}/include:${LD_LIBRARY_PATH}
export PATH=${CUDA_HOME}/bin:${PATH}
sudo apt-get install -y ninja-build
apt-get update && apt-get install -y ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6
apt-get clean
rm -rf /var/lib/apt/lists/*
- name: Install Pillow
run: python -m pip install Pillow==6.2.2
if: ${{matrix.torchvision < 0.5}}
- name: Install PyTorch
run: pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
run: python -m pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
- name: Install dependencies for compiling onnx when python=3.9
run: python -m pip install protobuf && apt-get update && apt-get -y install libprotobuf-dev protobuf-compiler cmake
if: ${{matrix.python-version == '3.9-dev'}}
- name: Install mmseg dependencies
run: |
pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/${CUDA_SHORT/-/}/${{matrix.torch_version}}/index.html
pip install -r requirements.txt
python -V
python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu102/${{matrix.mmcv_link}}/index.html
python -m pip install pycocotools
python -m pip install -r requirements.txt
python -c 'import mmcv; print(mmcv.__version__)'
- name: Install dependencies for compiling onnx when python=3.9
run: pip install protobuf && sudo apt-get install libprotobuf-dev protobuf-compiler
if: ${{matrix.python-version == '3.9'}}
- name: Build and install
run: rm -rf .eggs && pip install -e .
run: |
rm -rf .eggs
python setup.py check -m -s
TORCH_CUDA_ARCH_LIST=7.0 pip install .
- name: Run unittests and generate coverage report
run: |
coverage run --branch --source mmseg -m pytest tests/
coverage xml
coverage report -m
# Only upload coverage report for python3.7 && pytorch1.5
- name: Upload coverage to Codecov
if: ${{matrix.torch == '1.5.0+cu101' && matrix.python-version == '3.7'}}
uses: codecov/codecov-action@v1.0.14
uses: codecov/codecov-action@v2
with:
file: ./coverage.xml
files: ./coverage.xml
flags: unittests
env_vars: OS,PYTHON
name: codecov-umbrella
Expand Down

0 comments on commit 36eb2d8

Please sign in to comment.