Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[CI] Fix crd dataset unitest #310

Closed
wants to merge 18 commits into from
Closed

[CI] Fix crd dataset unitest #310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
bef5358
fix ci for circle ci
Sep 13, 2022
d14b431
fix bug in test_metafiles
Sep 13, 2022
2dcca13
add pr_stage_test for github ci
Sep 13, 2022
59ca3c5
add multiple version
Sep 13, 2022
df69f58
fix ut
pppppM Sep 15, 2022
f841685
fix lint
pppppM Sep 15, 2022
5f6b80b
Temporarily skip dataset UT
pppppM Sep 15, 2022
03b40e5
update github ci
pppppM Sep 15, 2022
e7a0231
add github lint ci
pppppM Sep 15, 2022
ab8bdc1
install wheel
pppppM Sep 15, 2022
8f0b299
Merge branch 'dev-1.x' into ci
Oct 9, 2022
a86555b
remove timm from requirements
Oct 9, 2022
72674b2
install wheel when test on windows
Oct 9, 2022
adfeb78
fix error
Oct 9, 2022
94f853e
fix bug
Oct 9, 2022
b1b8abd
remove github windows ci
Oct 9, 2022
cc31b4a
fix device error of arch_params when DsnasDDP
Oct 9, 2022
b5cc49e
fix CRD dataset ut
Oct 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 25 additions & 35 deletions .circleci/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ jobs:
command: |
pip install interrogate
interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-magic --ignore-regex "__repr__" --fail-under 80 mmrazor

build_cpu:
parameters:
# The python version must match available image tags in
Expand All @@ -37,8 +36,6 @@ jobs:
type: string
torchvision:
type: string
mmcv:
type: string
docker:
- image: cimg/python:<< parameters.python >>
resource_class: large
Expand All @@ -58,20 +55,21 @@ jobs:
name: Install PyTorch
command: |
python -V
python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html
pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html
- when:
condition:
equal: [ "3.9.0", << parameters.python >> ]
equal: ["3.9.0", << parameters.python >>]
steps:
- run: pip install "protobuf <= 3.20.1" && sudo apt-get update && sudo apt-get -y install libprotobuf-dev protobuf-compiler cmake
- run:
name: Install mmrazor dependencies
command: |
python -m pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main
python -m pip install << parameters.mmcv >>
python -m pip install git+ssh://git@github.com/open-mmlab/mmclassification.git@dev-1.x
python -m pip install git+ssh://git@github.com/open-mmlab/mmdetection.git@dev-3.x
python -m pip install git+ssh://git@github.com/open-mmlab/mmsegmentation.git@dev-1.x
pip install git+https://github.com/open-mmlab/mmengine.git@main
pip install -U openmim
mim install 'mmcv >= 2.0.0rc1'
pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
pip install git+https://github.com/open-mmlab/mmsegmentation.git@dev-1.x
pip install -r requirements.txt
- run:
name: Build and install
Expand All @@ -80,10 +78,9 @@ jobs:
- run:
name: Run unittests
command: |
python -m coverage run --branch --source mmrazor -m pytest tests/
python -m coverage xml
python -m coverage report -m

coverage run --branch --source mmrazor -m pytest tests/
coverage xml
coverage report -m
build_cuda:
parameters:
torch:
Expand All @@ -94,33 +91,31 @@ jobs:
cudnn:
type: integer
default: 7
mmcv:
type: string
machine:
image: ubuntu-2004-cuda-11.4:202110-01
# docker_layer_caching: true
resource_class: gpu.nvidia.small
steps:
- checkout
- run:
# CLoning repos in VM since Docker doesn't have access to the private key
# Cloning repos in VM since Docker doesn't have access to the private key
name: Clone Repos
command: |
git clone -b main --depth 1 ssh://git@github.com/open-mmlab/mmengine.git /home/circleci/mmengine
git clone -b dev-3.x --depth 1 ssh://git@github.com/open-mmlab/mmdetection.git /home/circleci/mmdetection
git clone -b dev-1.x --depth 1 ssh://git@github.com/open-mmlab/mmclassification.git /home/circleci/mmclassification
git clone -b dev-1.x --depth 1 ssh://git@github.com/open-mmlab/mmsegmentation.git /home/circleci/mmsegmentation
git clone -b main --depth 1 https://github.com/open-mmlab/mmengine.git /home/circleci/mmengine
git clone -b dev-3.x --depth 1 https://github.com/open-mmlab/mmdetection.git /home/circleci/mmdetection
git clone -b dev-1.x --depth 1 https://github.com/open-mmlab/mmclassification.git /home/circleci/mmclassification
git clone -b dev-1.x --depth 1 https://github.com/open-mmlab/mmsegmentation.git /home/circleci/mmsegmentation
- run:
name: Build Docker image
command: |
docker build .circleci/docker -t mmrazor:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >>
docker run --gpus all -t -d -v /home/circleci/project:/mmrazor -v /home/circleci/mmengine:/mmengine -v /home/circleci/mmdetection:/mmdetection -v /home/circleci/mmclassification:/mmclassification -v /home/circleci/mmsegmentation:/mmsegmentation -w /mmrazor --name mmrazor mmrazor:gpu
- run:
name: Install mmrazor dependencies
# pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu101/torch${{matrix.torch_version}}/index.html
command: |
docker exec mmrazor pip install -e /mmengine
docker exec mmrazor pip install << parameters.mmcv >>
docker exec mmrazor pip install -U openmim
docker exec mmrazor mim install 'mmcv >= 2.0.0rc1'
docker exec mmrazor pip install -e /mmdetection
docker exec mmrazor pip install -e /mmclassification
docker exec mmrazor pip install -e /mmsegmentation
Expand All @@ -132,7 +127,7 @@ jobs:
- run:
name: Run unittests
command: |
docker exec mmrazor python -m pytest tests/
docker exec mmrazor pytest tests/

workflows:
pr_stage_lint:
Expand All @@ -144,10 +139,10 @@ workflows:
branches:
ignore:
- dev-1.x
- 1.x
pr_stage_test:
when:
not:
<< pipeline.parameters.lint_only >>
not: << pipeline.parameters.lint_only >>
jobs:
- lint:
name: lint
Expand All @@ -159,16 +154,14 @@ workflows:
name: minimum_version_cpu
torch: 1.6.0
torchvision: 0.7.0
python: 3.6.9 # The lowest python 3.6.x version available on CircleCI images
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cpu/torch1.6.0/mmcv_full-2.0.0rc1-cp36-cp36m-manylinux1_x86_64.whl
python: 3.6.9 # The lowest python 3.6.x version available on CircleCI images
requires:
- lint
- build_cpu:
name: maximum_version_cpu
torch: 1.9.0
torchvision: 0.10.0
torch: 1.12.1
torchvision: 0.13.1
python: 3.9.0
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cpu/torch1.9.0/mmcv_full-2.0.0rc1-cp39-cp39-manylinux1_x86_64.whl
requires:
- minimum_version_cpu
- hold:
Expand All @@ -181,20 +174,17 @@ workflows:
# Use double quotation mark to explicitly specify its type
# as string instead of number
cuda: "10.2"
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cu102/torch1.8.0/mmcv_full-2.0.0rc1-cp37-cp37m-manylinux1_x86_64.whl
requires:
- hold
merge_stage_test:
when:
not:
<< pipeline.parameters.lint_only >>
not: << pipeline.parameters.lint_only >>
jobs:
- build_cuda:
name: minimum_version_gpu
torch: 1.6.0
# Use double quotation mark to explicitly specify its type
# as string instead of number
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cu101/torch1.6.0/mmcv_full-2.0.0rc1-cp37-cp37m-manylinux1_x86_64.whl
cuda: "10.1"
filters:
branches:
Expand Down
201 changes: 201 additions & 0 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,201 @@
name: build

on:
push:
paths-ignore:
- "README.md"
- "README_zh-CN.md"
- "model-index.yml"
- "configs/**"
- "docs/**"
- ".dev_scripts/**"

pull_request:
paths-ignore:
- "README.md"
- "README_zh-CN.md"
- "docs/**"
- "demo/**"
- ".dev_scripts/**"
- ".circleci/**"

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
test_linux:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
torch: [1.6.0, 1.7.0, 1.8.0, 1.9.0, 1.10.0, 1.11.0, 1.12.0]
include:
- torch: 1.6.0
torch_version: 1.6
torchvision: 0.7.0
- torch: 1.7.0
torch_version: 1.7
torchvision: 0.8.1
- torch: 1.7.0
torch_version: 1.7
torchvision: 0.8.1
python-version: 3.8
- torch: 1.8.0
torch_version: 1.8
torchvision: 0.9.0
- torch: 1.8.0
torch_version: 1.8
torchvision: 0.9.0
python-version: 3.8
- torch: 1.9.0
torch_version: 1.9
torchvision: 0.10.0
- torch: 1.9.0
torch_version: 1.9
torchvision: 0.10.0
python-version: 3.8
- torch: 1.10.0
torch_version: 1.10
torchvision: 0.11.0
- torch: 1.10.0
torch_version: 1.10
torchvision: 0.11.0
python-version: 3.8
- torch: 1.11.0
torch_version: 1.11
torchvision: 0.12.0
- torch: 1.11.0
torch_version: 1.11
torchvision: 0.12.0
python-version: 3.8
- torch: 1.12.0
torch_version: 1.12
torchvision: 0.13.0
- torch: 1.12.0
torch_version: 1.12
torchvision: 0.13.0
python-version: 3.8

steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: |
pip install pip --upgrade
pip install wheel
- name: Install PyTorch
run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
- name: Install MMEngine
run: pip install git+https://github.com/open-mmlab/mmengine.git@main
- name: Install MMCV
run: |
pip install -U openmim
mim install 'mmcv >= 2.0.0rc1'
- name: Install MMCls
run: pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
- name: Install MMDet
run: pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
- name: Install MMSeg
run: pip install git+https://github.com/open-mmlab/mmsegmentation.git@dev-1.x
- name: Install other dependencies
run: pip install -r requirements.txt
- name: Build and install
run: rm -rf .eggs && pip install -e .
- name: Run unittests and generate coverage report
run: |
coverage run --branch --source mmrazor -m pytest tests/
coverage xml
coverage report -m
# Upload coverage report for python3.8 && pytorch1.12.0 cpu
- name: Upload coverage to Codecov
if: ${{matrix.torch == '1.12.0' && matrix.python-version == '3.8'}}
uses: codecov/codecov-action@v2
with:
file: ./coverage.xml
flags: unittests
env_vars: OS,PYTHON
name: codecov-umbrella
fail_ci_if_error: false



test_cuda:
runs-on: ubuntu-18.04
container:
image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel
strategy:
matrix:
python-version: [3.7]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: pip install pip --upgrade
- name: Fetch GPG keys
run: |
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
- name: Install Python-dev
run: apt-get update && apt-get install -y python${{matrix.python-version}}-dev
if: ${{matrix.python-version != 3.9}}
- name: Install system dependencies
run: |
apt-get update
apt-get install -y ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libxrender-dev
- name: Install mmrazor dependencies
run: |
pip install git+https://github.com/open-mmlab/mmengine.git@main
pip install -U openmim
mim install 'mmcv >= 2.0.0rc1'
pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
pip install git+https://github.com/open-mmlab/mmsegmentation.git@dev-1.x
pip install -r requirements.txt
- name: Build and install
run: |
python setup.py check -m -s
TORCH_CUDA_ARCH_LIST=7.0 pip install -e .

# test_windows:
# runs-on: ${{ matrix.os }}
# strategy:
# matrix:
# os: [windows-2022]
# python: [3.7]
# platform: [cpu]
# steps:
# - uses: actions/checkout@v2
# - name: Set up Python ${{ matrix.python-version }}
# uses: actions/setup-python@v2
# with:
# python-version: ${{ matrix.python-version }}
# - name: Upgrade pip
# run: |
# pip install pip --upgrade
# pip install wheel
# - name: Install lmdb
# run: pip install lmdb
# - name: Install PyTorch
# run: pip install torch==1.8.1+${{matrix.platform}} torchvision==0.9.1+${{matrix.platform}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
# - name: Install mmrazor dependencies
# run: |
# pip install git+https://github.com/open-mmlab/mmengine.git@main
# pip install -U openmim
# mim install 'mmcv >= 2.0.0rc1'
# pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
# pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
# pip install git+https://github.com/open-mmlab/mmsegmentation.git@dev-1.x
# pip install -r requirements.txt
# - name: Build and install
# run: |
# pip install -e .
# - name: Run unittests and generate coverage report
# run: |
# pytest tests/
27 changes: 27 additions & 0 deletions .github/workflows/lint.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
name: lint

on: [push, pull_request]

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.7
uses: actions/setup-python@v2
with:
python-version: 3.7
- name: Install pre-commit hook
run: |
pip install pre-commit
pre-commit install
- name: Linting
run: pre-commit run --all-files
- name: Check docstring coverage
run: |
pip install interrogate
interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-regex "__repr__" --fail-under 80 mmrazor
Loading