From 01f2c4b7fe017d092a267fe2a31a0b27d7f76d79 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Detlefsen Date: Wed, 24 Apr 2024 11:16:31 +0200 Subject: [PATCH] Description on how to run tests & build docs (#2500) * readme tests * add note on make * add pool skipper * update Make docs * Apply suggestions from code review --------- Co-authored-by: Jirka Borovec <6035284+Borda@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/docs-build.yml | 12 ++------ Makefile | 9 +++--- tests/README.md | 35 ++++++++++++++++++++++++ tests/unittests/__init__.py | 2 ++ tests/unittests/bases/test_ddp.py | 7 ++++- tests/unittests/conftest.py | 12 ++------ tests/unittests/text/_helpers.py | 4 ++- tests/unittests/wrappers/test_running.py | 3 +- 8 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 tests/README.md diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index d0114cb8070..154dc88b413 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -56,18 +56,12 @@ jobs: sudo apt-get install -y \ texlive-latex-extra texlive-pictures texlive-fonts-recommended dvipng cm-super - - name: Pull sphinx template - run: | - pip install awscli - aws s3 sync --no-sign-request s3://sphinx-packages/ dist/ - pip install lai-sphinx-theme -U -f dist/ - name: Install package & dependencies run: | - pip --version + make get-sphinx-template pip install . -U -r requirements/_docs.txt \ - --find-links="dist/" --find-links="${PYPI_CACHE}" --find-links="${TORCH_URL}" - pip list - + --find-links="${PYPI_CACHE}" --find-links="${TORCH_URL}" + - run: pip list - name: Full build for deployment if: github.event_name != 'pull_request' run: echo "SPHINX_FETCH_ASSETS=1" >> $GITHUB_ENV diff --git a/Makefile b/Makefile index ea5bc674fb5..ebe2503ca4b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean test pull-template docs env data +.PHONY: clean test get-sphinx-template docs env data export FREEZE_REQUIREMENTS=1 # assume you have installed need packages @@ -28,12 +28,13 @@ test: clean env data cd tests && python -m pytest unittests -v --cov=torchmetrics cd tests && python -m coverage report -pull-template: +get-sphinx-template: pip install -q awscli aws s3 sync --no-sign-request s3://sphinx-packages/ dist/ + pip install lai-sphinx-theme -q -U -f dist/ -docs: clean pull-template - pip install -e . --quiet -r requirements/_docs.txt -f dist/ +docs: clean get-sphinx-template + pip install -e . --quiet -r requirements/_docs.txt # apt-get install -y texlive-latex-extra dvipng texlive-pictures texlive-fonts-recommended cm-super cd docs && make html --debug --jobs $(nproc) SPHINXOPTS="-W --keep-going" diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000000..7f5cbd4e98a --- /dev/null +++ b/tests/README.md @@ -0,0 +1,35 @@ +# Running tests locally + +To run the tests locally, you need to have the full development environment set up. This can be setup by running +the following command in the root directory of the project: + +```bash +pip install . -r requirements/_devel.txt +``` + +Then for windows users, to execute the tests (unit tests and integration tests) run the following command (will only run non-DDP tests): + +```bash +pytest tests/ +``` + +For linux/Mac users you will need to provide the `-m` argument to indicate if `ddp` tests should also be executed: + +```bash +pytest -m DDP tests/ # to run only DDP tests +pytest -m "not DDP" tests/ # to run all tests except DDP tests +``` + +## Simply Make + +Alternatively, for Unix with `make` installed, simply running `make test` from the root of the project will install +all requirements and run the full test suit. + +## Test particular domain + +To run only unittests, point the command only to the `tests/unittests` directory. Similarly, to only run a subset of the +unittests, like all tests related to the regression domain, run the following command: + +```bash +pytest tests/unittests/regression/ +``` diff --git a/tests/unittests/__init__.py b/tests/unittests/__init__.py index dda7e8b3920..6cc99ce84a4 100644 --- a/tests/unittests/__init__.py +++ b/tests/unittests/__init__.py @@ -14,6 +14,7 @@ NUM_CLASSES, NUM_PROCESSES, THRESHOLD, + USE_PYTEST_POOL, setup_ddp, ) @@ -60,5 +61,6 @@ class _GroupInput(NamedTuple): "NUM_CLASSES", "NUM_PROCESSES", "THRESHOLD", + "USE_PYTEST_POOL", "setup_ddp", ] diff --git a/tests/unittests/bases/test_ddp.py b/tests/unittests/bases/test_ddp.py index e927816e203..c057d0cbdf8 100644 --- a/tests/unittests/bases/test_ddp.py +++ b/tests/unittests/bases/test_ddp.py @@ -24,7 +24,7 @@ from torchmetrics.utilities.exceptions import TorchMetricsUserError from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_2_1 -from unittests import NUM_PROCESSES +from unittests import NUM_PROCESSES, USE_PYTEST_POOL from unittests._helpers import seed_all from unittests._helpers.testers import DummyListMetric, DummyMetric, DummyMetricSum @@ -88,6 +88,7 @@ def _test_ddp_compositional_tensor(rank: int, worldsize: int = NUM_PROCESSES) -> @pytest.mark.DDP() @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows") +@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.") @pytest.mark.parametrize( "process", [ @@ -125,6 +126,7 @@ def compute(self): @pytest.mark.DDP() @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows") +@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.") def test_non_contiguous_tensors(): """Test that gather_all operation works for non-contiguous tensors.""" pytest.pool.map(_test_non_contiguous_tensors, range(NUM_PROCESSES)) @@ -232,6 +234,7 @@ def reload_state_dict(state_dict, expected_x, expected_c): @pytest.mark.DDP() @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows") +@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.") def test_state_dict_is_synced(tmpdir): """Tests that metrics are synced while creating the state dict but restored after to continue accumulation.""" pytest.pool.map(partial(_test_state_dict_is_synced, tmpdir=tmpdir), range(NUM_PROCESSES)) @@ -260,6 +263,7 @@ def _test_sync_on_compute_list_state(rank, sync_on_compute): @pytest.mark.DDP() @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows") +@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.") @pytest.mark.parametrize("sync_on_compute", [True, False]) @pytest.mark.parametrize("test_func", [_test_sync_on_compute_list_state, _test_sync_on_compute_tensor_state]) def test_sync_on_compute(sync_on_compute, test_func): @@ -276,6 +280,7 @@ def _test_sync_with_empty_lists(rank): @pytest.mark.DDP() @pytest.mark.skipif(not _TORCH_GREATER_EQUAL_2_1, reason="test only works on newer torch versions") @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows") +@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.") def test_sync_with_empty_lists(): """Test that synchronization of states can be enabled and disabled for compute.""" pytest.pool.map(_test_sync_with_empty_lists, range(NUM_PROCESSES)) diff --git a/tests/unittests/conftest.py b/tests/unittests/conftest.py index d66ca24225c..f09f884adeb 100644 --- a/tests/unittests/conftest.py +++ b/tests/unittests/conftest.py @@ -60,11 +60,7 @@ def setup_ddp(rank, world_size): def pytest_sessionstart(): - """Global initialization of multiprocessing pool. - - Runs before any test. - - """ + """Global initialization of multiprocessing pool; runs before any test.""" if not USE_PYTEST_POOL: return pool = Pool(processes=NUM_PROCESSES) @@ -73,11 +69,7 @@ def pytest_sessionstart(): def pytest_sessionfinish(): - """Correctly closes the global multiprocessing pool. - - Runs after all tests. - - """ + """Correctly closes the global multiprocessing pool; runs after all tests.""" if not USE_PYTEST_POOL: return pytest.pool.close() diff --git a/tests/unittests/text/_helpers.py b/tests/unittests/text/_helpers.py index 1f237d3e96f..e55944a72af 100644 --- a/tests/unittests/text/_helpers.py +++ b/tests/unittests/text/_helpers.py @@ -22,7 +22,7 @@ from torch import Tensor from torchmetrics import Metric -from unittests import NUM_PROCESSES, _reference_cachier +from unittests import NUM_PROCESSES, USE_PYTEST_POOL, _reference_cachier from unittests._helpers import seed_all from unittests._helpers.testers import ( MetricTester, @@ -369,6 +369,8 @@ def run_class_metric_test( if ddp: if sys.platform == "win32": pytest.skip("DDP not supported on windows") + if not USE_PYTEST_POOL: + pytest.skip("DDP pool is not available.") pytest.pool.starmap( partial(_class_test, **common_kwargs, **kwargs_update), [(rank, NUM_PROCESSES) for rank in range(NUM_PROCESSES)], diff --git a/tests/unittests/wrappers/test_running.py b/tests/unittests/wrappers/test_running.py index 32319aa8fd7..ecac751453d 100644 --- a/tests/unittests/wrappers/test_running.py +++ b/tests/unittests/wrappers/test_running.py @@ -22,7 +22,7 @@ from torchmetrics.regression import MeanAbsoluteError, MeanSquaredError, PearsonCorrCoef from torchmetrics.wrappers import Running -from unittests import NUM_PROCESSES +from unittests import NUM_PROCESSES, USE_PYTEST_POOL def test_errors_on_wrong_input(): @@ -145,6 +145,7 @@ def _test_ddp_running(rank, dist_sync_on_step, expected): @pytest.mark.DDP() @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows") +@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.") @pytest.mark.parametrize(("dist_sync_on_step", "expected"), [(False, 1), (True, 2)]) def test_ddp_running(dist_sync_on_step, expected): """Check that the dist_sync_on_step gets correctly passed to base metric."""