forked from facebookresearch/habitat-lab
-
Notifications
You must be signed in to change notification settings - Fork 0
382 lines (382 loc) · 15.5 KB
/
install_and_test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
name: facebookresearch/habitat-lab/install_and_test
on:
pull_request:
branches:
- main
schedule:
- cron: 0 7 * * *
push:
tags:
- v**
env:
CODECOV_TOKEN: xxxx91d3
jobs:
set-date:
runs-on: ubuntu-latest
outputs:
date: ${{ steps.set_date.outputs.date }}
steps:
- name: Combine precommit config and python versions for caching
id: set_date
run: |-
echo "date=$(date +%F)" >> "$GITHUB_OUTPUT"
pre-commit:
runs-on: ubuntu-latest
needs: set-date
steps:
- uses: actions/checkout@v4.1.1
- name: Setup python
uses: actions/setup-python@v5.0.0
with:
python-version: '3.9.16'
- name: Combine precommit config and python versions for caching
run: |-
cat .pre-commit-config.yaml > pre-commit-deps.txt
python -VV >> pre-commit-deps.txt
- name: restore_cache
uses: actions/cache@v3.3.2
with:
key: v1-precommit-deps-{{ hashFiles('pre-commit-deps.tx') }}-{{ needs.set-date.outputs.date }}
path: "~/.cache/pre-commit"
- name: Install Dependencies
run: |-
pip install -U pip setuptools pre-commit
# Install the hooks now so that they'll be cached
pre-commit install-hooks
- name: Check Code Style using pre-commit
run: SKIP=clang-format,eslint pre-commit run --show-diff-on-failure --all-files
python_lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.0
- name: Setup python
uses: actions/setup-python@v5.0.0
with:
python-version: '3.9.16'
- name: setup
run: |-
pip install black==23.1.0 --progress-bar off
pip install "isort[pyproject]" numpy --progress-bar off
pip install mypy==0.991 types-mock types-Pillow types-tqdm types-PyYAML --progress-bar off
pip install -r habitat-lab/requirements.txt --progress-bar off
- name: run black
run: |-
black --exclude '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist)|examples/tutorials/(notebooks|nb_python)' habitat-lab/. habitat-baselines/. examples/. test/. --diff
black --exclude '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist)|examples/tutorials/(notebooks|nb_python)' habitat-lab/. habitat-baselines/. examples/. test/. --check
- name: run isort
run: |-
isort --version
isort habitat-lab/. habitat-baselines/. examples/. test/. --diff
isort habitat-lab/. habitat-baselines/. examples/. test/. --check-only
- name: run mypy
run: |-
mypy --version
mypy --exclude="^docs/|setup.py$"
- name: run assert no files in habitat and habitat_baselines
run: |-
if test -d habitat ; then echo "folder habitat should not exist"; exit 1; fi
if test -d habitat_baselines ; then echo "folder habitat_baselines should not exist"; exit 1; fi
install_and_test_ubuntu:
runs-on: 4-core-ubuntu-gpu-t4
env:
FPS_THRESHOLD: 900
steps:
- uses: actions/checkout@v4.1.0
with:
path: "./habitat-lab"
- name: Install dependencies
run: |-
sudo apt-get update || true
sudo apt-get install -y --no-install-recommends \
build-essential \
git \
curl \
vim \
ca-certificates \
libjpeg-dev \
libglm-dev \
libegl1-mesa-dev \
ninja-build \
xorg-dev \
freeglut3-dev \
pkg-config \
wget \
zip \
lcov\
libhdf5-dev \
libomp-dev \
unzip \
cmake
sudo apt install -y --allow-change-held-packages \
texlive-base \
texlive-latex-extra \
texlive-fonts-extra \
texlive-fonts-recommended
- name: Free up disk space
if: ${{ runner.os == 'Linux' }}
# https://github.com/easimon/maximize-build-space/blob/master/action.yml
# https://github.com/easimon/maximize-build-space/tree/test-report
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
- name: Setup python
uses: actions/setup-python@v5.0.0
with:
python-version: '3.9.16'
- name: Install cuda
run: |-
echo "Install cuda"
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
sudo apt-get update
sudo apt-get -y install cuda-toolkit-12-3
shell: bash
- name: Check CUDA
run: |
nvidia-smi
- name: Cache conda
uses: actions/cache@v3.3.2
with:
key: conda-{{ hashFiles('habitat-lab/.circleci/config.yml') }}-{{ needs.set-date.outputs.date }}
path: "~/miniconda"
- name: Install conda and dependencies
run: |-
if [ ! -d ~/miniconda ]
then
curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
chmod +x ~/miniconda.sh
bash ~/miniconda.sh -b -p $HOME/miniconda
rm ~/miniconda.sh
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
conda create -y -n habitat python=3.9
. activate habitat
conda install -q -y mkl==2021.4.0
conda install -q -y -c conda-forge ninja ccache
pip install pytest-sugar>=0.9.6 mock cython pygame flaky pytest pytest-mock pytest-cov psutil
fi
- name: Install pytorch
run: |-
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat;
if [ ! -f ~/miniconda/pytorch_installed ]
then
# For whatever reason we have to install pytorch first. If it isn't
# it installs the 1.4 cpuonly version. Which is no good.
echo "Installing pytorch"
conda install -y pytorch==1.12.1=py3.9_cuda11.3_cudnn8.3.2_0 torchvision==0.13.1=py39_cu113 cudatoolkit=11.3 -c pytorch -c nvidia
fi
touch ~/miniconda/pytorch_installed
python -c 'import torch; print("Has cuda?", torch.cuda.is_available()); print("torch version:", torch.__version__);'
- name: restore_cache
uses: actions/cache@v3.3.2
with:
key: v1-habitat-sim-{{ hashFiles('./hsim_sha') }}-{{ needs.set-date.outputs.date }}
path: "./habitat-sim"
- name: Cache ccache
uses: actions/cache@v3.3.2
with:
key: ccache-{{ arch }}-main-{{ needs.set-date.outputs.date }}
path: .ccache
- name: CCache initialization
run: |-
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat;
ccache --show-stats
ccache --zero-stats
ccache --max-size=10.0G
- name: Build and install habitat-sim
run: |-
if [ ! -d ./habitat-sim ]
then
git clone https://github.com/facebookresearch/habitat-sim.git --recursive
fi
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat;
cd habitat-sim
pip install -r requirements.txt --progress-bar off
pip install imageio imageio-ffmpeg
python -u setup.py install --headless --with-cuda --bullet
- name: Ccache stats
run: |-
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat;
ccache --show-stats
- name: Download test data
run: |-
sudo apt install git-lfs
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat
git lfs install
conda install -y gitpython git-lfs
python -m habitat_sim.utils.datasets_download --uids ci_test_assets franka_panda hab_spot_arm --data-path habitat-sim/data/ --no-replace --no-prune
- name: Run sim benchmark
run: |-
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat; cd habitat-sim
python examples/example.py --scene data/scene_datasets/habitat-test-scenes/van-gogh-room.glb --silent --test_fps_regression $FPS_THRESHOLD
- name: Install api with baselines
run: |-
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat; cd habitat-lab
ln -s ../habitat-sim/data data
while [ ! -f ~/miniconda/pytorch_installed ]; do sleep 2; done # wait for Pytorch
pip install -e habitat-lab
pip install -e habitat-baselines
- name: Display dependency versions
run: |-
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat
export CONDA_ENV_DUMP=conda_env_dump.log
echo "pip freeze" | tee -a $CONDA_ENV_DUMP
pip freeze | tee -a $CONDA_ENV_DUMP
echo "###########" | tee -a $CONDA_ENV_DUMP
echo "conda list" | tee -a $CONDA_ENV_DUMP
conda list | tee -a $CONDA_ENV_DUMP
echo "###########" | tee -a $CONDA_ENV_DUMP
echo "habitat-sim commit" | tee -a $CONDA_ENV_DUMP
cat ./hsim_sha | tee -a $CONDA_ENV_DUMP
- uses: actions/upload-artifact@v4.0.0
with:
path: conda_env_dump.log
- name: Run api tests
run: |-
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat; cd habitat-lab
export PYTHONPATH=.:$PYTHONPATH
export MULTI_PROC_OFFSET=0 && export MAGNUM_LOG=quiet && export HABITAT_SIM_LOG=quiet
python -m pytest --cov-report=xml --cov-report term --cov=./
# # This item has no matching transformer
# - codecov_codecov_upload:
- name: Run baseline training tests
run: |-
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat; cd habitat-lab
export PYTHONPATH=.:$PYTHONPATH
export MULTI_PROC_OFFSET=0 && export MAGNUM_LOG=quiet && export HABITAT_SIM_LOG=quiet
# This is a flag that enables test_test_baseline_training to work
export TEST_BASELINE_SMALL=1
python -m pytest test/test_baseline_training.py -s
- name: Run Hab2.0 benchmark
run: |-
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat; cd habitat-lab
python -m habitat_sim.utils.datasets_download --uids hab2_bench_assets
mkdir -p data/ep_datasets/
cp data/hab2_bench_assets/bench_scene.json.gz data/ep_datasets/
bash scripts/hab2_bench/bench_runner.sh
python scripts/hab2_bench/plot_bench.py
# Assert the SPS number are up to standard
python scripts/hab2_bench/assert_bench.py
- name: Build api documentation
run: |-
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate habitat; cd habitat-lab
# Download sim inventory for crosslinking (no need to build
# the whole sim docs for that)
# TODO: take it from github.com/facebookmicrosites/habitat-website
# instead
mkdir -p ../habitat-sim/build/docs-public/habitat-sim
curl -s https://aihabitat.org/docs/habitat-sim/objects.inv > ../habitat-sim/build/docs-public/habitat-sim/objects.inv
cd docs
conda install -y -c conda-forge doxygen=1.9.5
conda install -y jinja2 pygments docutils
mkdir -p ../build/docs
./build-public.sh
- name: Ensure non-editable mode works
run: |-
cd habitat-lab
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
conda create -y -n non-editable-install python=3.9
. activate non-editable-install
conda install -y -c conda-forge -c aihabitat-nightly habitat-sim
pip install habitat-lab/
python -c 'import habitat; print("habitat version:", habitat.__version__)'
pip install habitat-baselines/
python -c 'import habitat_baselines; print("habitat_baselines version:", habitat_baselines.__version__)'
- name: Build sdist and bdist
run: |-
cd habitat-lab
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
conda create -y -n build-env python=3.9
. activate build-env
pip install --upgrade build
python -m build -s -w -C--global-option=egg_info -C--global-option=--tag-date habitat-lab/
python -m build -s -w -C--global-option=egg_info -C--global-option=--tag-date habitat-baselines/
- name: Ensure sdist and bdist intall work
run: |-
cd habitat-lab
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
conda create -y -n bdist-install python=3.9
. activate bdist-install
conda install -y -c conda-forge -c aihabitat-nightly habitat-sim
conda create -n sdist-install --clone bdist-install
# install from built distribution:
. activate bdist-install
pip install habitat-lab/dist/habitat_lab*.whl
python -c 'import habitat; print("habitat version:", habitat.__version__)'
pip install habitat-baselines/dist/habitat_baselines*.whl
python -c 'import habitat_baselines; print("habitat_baselines version:", habitat_baselines.__version__)'
# install from source distribution:
. activate sdist-install
pip install habitat-lab/dist/habitat-lab*.tar.gz
python -c 'import habitat; print("habitat version:", habitat.__version__)'
pip install habitat-baselines/dist/habitat-baselines*.tar.gz
python -c 'import habitat_baselines; print("habitat_baselines version:", habitat_baselines.__version__)'
- uses: actions/upload-artifact@v4.0.0
with:
path: habitat-lab/data/profile
pypi_deploy:
if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || github.event_name == 'schedule'
runs-on: ubuntu-latest
needs:
- pre-commit
- python_lint
- install_and_test_ubuntu
env:
FPS_THRESHOLD: 900
username_env_var: TESTPYPI_USERNAME
password_env_var: TESTPYPI_PASSWORD
repository: testpypi
PYPI_PASSWORD:
PYPI_USERNAME:
TESTPYPI_PASSWORD:
TESTPYPI_USERNAME:
steps:
- uses: actions/checkout@v4.1.0
with:
path: "./habitat-lab"
- name: Install conda
run: |-
if [ ! -d ~/miniconda ]
then
curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
chmod +x ~/miniconda.sh
bash ~/miniconda.sh -b -p $HOME/miniconda
rm ~/miniconda.sh
fi
- name: Build sdist and bdist
run: |-
cd habitat-lab
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
conda create -y -n build-env python=3.9
. activate build-env
pip install --upgrade build
python -m build -s -w -C--global-option=egg_info -C--global-option=--tag-date habitat-lab/
python -m build -s -w -C--global-option=egg_info -C--global-option=--tag-date habitat-baselines/
- name: Deploy Habitat-Lab and Habitat-Baselines distributions to PyPI
run: |-
cd habitat-lab
export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
. activate build-env
pip install --upgrade twine
twine upload \
--username ${${{ env.username_env_var }}} \
--password ${${{ env.password_env_var }}} \
--repository ${{ env.repository }} \
habitat-lab/dist/*
twine upload \
--username ${${{ env.username_env_var }}} \
--password ${${{ env.password_env_var }}} \
--repository ${{ env.repository }} \
habitat-baselines/dist/*