From 0f6eee357e9d26b39510f55e80c47e005b568ad0 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:55:32 -0500 Subject: [PATCH 1/6] implement `XCLIM_TESTDATA_BRANCH` and `XCLIM_PREFETCH_TESTING_DATA` environment variables, remove obsolete subset-specific testing data, adjust testing data fetching logic --- xclim/testing/tests/conftest.py | 27 +-- xclim/testing/tests/data.py | 12 +- xclim/testing/tests/data/eastern_canada.json | 40 ----- xclim/testing/tests/data/meridian.json | 44 ----- xclim/testing/tests/data/meridian_multi.json | 67 ------- xclim/testing/tests/data/multi_regions.json | 168 ------------------ xclim/testing/tests/data/poslons.json | 48 ----- xclim/testing/tests/data/small_geojson.json | 40 ----- .../tests/data/southern_qc_geojson.json | 52 ------ xclim/testing/tests/test_ensembles.py | 4 +- 10 files changed, 21 insertions(+), 481 deletions(-) delete mode 100644 xclim/testing/tests/data/eastern_canada.json delete mode 100644 xclim/testing/tests/data/meridian.json delete mode 100644 xclim/testing/tests/data/meridian_multi.json delete mode 100644 xclim/testing/tests/data/multi_regions.json delete mode 100644 xclim/testing/tests/data/poslons.json delete mode 100644 xclim/testing/tests/data/small_geojson.json delete mode 100644 xclim/testing/tests/data/southern_qc_geojson.json diff --git a/xclim/testing/tests/conftest.py b/xclim/testing/tests/conftest.py index 7c1ccf3f5..baa7e7188 100644 --- a/xclim/testing/tests/conftest.py +++ b/xclim/testing/tests/conftest.py @@ -22,8 +22,8 @@ from xclim.testing.utils import _default_cache_dir # noqa from xclim.testing.utils import open_dataset as _open_dataset -MAIN_TESTDATA_BRANCH = os.getenv("MAIN_TESTDATA_BRANCH", "main") -SKIP_TEST_DATA = os.getenv("SKIP_TEST_DATA") +TESTDATA_BRANCH = os.getenv("XCLIM_TESTDATA_BRANCH", "main") +PREFETCH_TESTING_DATA = os.getenv("XCLIM_PREFETCH_TESTING_DATA") @pytest.fixture @@ -518,7 +518,7 @@ def cmip3_day_tas(threadsafe_data_dir): ds = _open_dataset( "cmip3/tas.sresb1.giss_model_e_r.run1.atm.da.nc", cache_dir=threadsafe_data_dir, - branch=MAIN_TESTDATA_BRANCH, + branch=TESTDATA_BRANCH, ) yield ds.tas ds.close() @@ -527,7 +527,7 @@ def cmip3_day_tas(threadsafe_data_dir): @pytest.fixture(scope="session") def open_dataset(threadsafe_data_dir): def _open_session_scoped_file( - file: str | os.PathLike, branch: str = MAIN_TESTDATA_BRANCH, **xr_kwargs + file: str | os.PathLike, branch: str = TESTDATA_BRANCH, **xr_kwargs ): return _open_dataset( file, cache_dir=threadsafe_data_dir, branch=branch, **xr_kwargs @@ -544,7 +544,7 @@ def add_imports(xdoctest_namespace, threadsafe_data_dir) -> None: ns["xr"] = xclim.testing # xr.open_dataset(...) -> xclim.testing.open_dataset(...) ns["xclim"] = xclim ns["open_dataset"] = partial( - _open_dataset, cache_dir=threadsafe_data_dir, branch=MAIN_TESTDATA_BRANCH + _open_dataset, cache_dir=threadsafe_data_dir, branch=TESTDATA_BRANCH ) # Needed for modules where xarray is imported as `xr` @@ -583,7 +583,7 @@ def atmosds(threadsafe_data_dir) -> xr.Dataset: return _open_dataset( threadsafe_data_dir.joinpath("atmosds.nc"), cache_dir=threadsafe_data_dir, - branch=MAIN_TESTDATA_BRANCH, + branch=TESTDATA_BRANCH, ) @@ -610,16 +610,19 @@ def gather_session_data(threadsafe_data_dir, worker_id, xdoctest_namespace): When running pytest with multiple workers, one worker will copy data remotely to _default_cache_dir while other workers wait using lockfile. Once the lock is released, all workers will copy data to their local threadsafe_data_dir.""" - if worker_id == "master": - if not SKIP_TEST_DATA: - populate_testing_data(branch=MAIN_TESTDATA_BRANCH) - else: - if not SKIP_TEST_DATA: + + if ( + not _default_cache_dir.joinpath(TESTDATA_BRANCH).exists() + or PREFETCH_TESTING_DATA + ): + if worker_id in "master": + populate_testing_data(branch=TESTDATA_BRANCH) + else: _default_cache_dir.mkdir(exist_ok=True) test_data_being_written = FileLock(_default_cache_dir.joinpath(".lock")) with test_data_being_written as fl: # This flag prevents multiple calls from re-attempting to download testing data in the same pytest run - populate_testing_data(branch=MAIN_TESTDATA_BRANCH) + populate_testing_data(branch=TESTDATA_BRANCH) _default_cache_dir.joinpath(".data_written").touch() fl.acquire() shutil.copytree(_default_cache_dir, threadsafe_data_dir) diff --git a/xclim/testing/tests/data.py b/xclim/testing/tests/data.py index e8b60d432..b5a2c1562 100644 --- a/xclim/testing/tests/data.py +++ b/xclim/testing/tests/data.py @@ -17,7 +17,7 @@ from xclim.testing import open_dataset as _open_dataset from xclim.testing.utils import _default_cache_dir # noqa -MAIN_TESTDATA_BRANCH = os.getenv("MAIN_TESTDATA_BRANCH", "main") +TESTDATA_BRANCH = os.getenv("XCLIM_TESTDATA_BRANCH", "main") TD = Path(__file__).parent / "data" @@ -34,7 +34,7 @@ def generate_atmos(cache_dir: Path): with _open_dataset( "ERA5/daily_surface_cancities_1990-1993.nc", cache_dir=cache_dir, - branch=MAIN_TESTDATA_BRANCH, + branch=TESTDATA_BRANCH, ) as ds: tn10 = calendar.percentile_doy(ds.tasmin, per=10) t10 = calendar.percentile_doy(ds.tas, per=10) @@ -60,7 +60,7 @@ def generate_atmos(cache_dir: Path): def populate_testing_data( temp_folder: Path | None = None, - branch: str = MAIN_TESTDATA_BRANCH, + branch: str = TESTDATA_BRANCH, _local_cache: Path = _default_cache_dir, ): """Perform calls to GitHub for the relevant testing data.""" @@ -112,8 +112,6 @@ def add_example_file_paths(cache_dir: Path) -> dict[str]: ns["path_to_tasmax_file"] = "NRCANdaily/nrcan_canada_daily_tasmax_1990.nc" ns["path_to_tasmin_file"] = "NRCANdaily/nrcan_canada_daily_tasmin_1990.nc" ns["path_to_tas_file"] = "ERA5/daily_surface_cancities_1990-1993.nc" - ns["path_to_multi_shape_file"] = str(TD / "multi_regions.json") - ns["path_to_shape_file"] = str(TD / "southern_qc_geojson.json") ns["path_to_ensemble_file"] = "EnsembleReduce/TestEnsReduceCriteria.nc" # For core.utils.load_module example @@ -151,9 +149,7 @@ def add_example_file_paths(cache_dir: Path) -> dict[str]: atmos_file = cache_dir.joinpath("atmosds.nc") # Give access to dataset variables by name in xdoctest namespace - with _open_dataset( - atmos_file, branch=MAIN_TESTDATA_BRANCH, cache_dir=cache_dir - ) as ds: + with _open_dataset(atmos_file, branch=TESTDATA_BRANCH, cache_dir=cache_dir) as ds: for variable in ds.data_vars: ns[f"{variable}_dataset"] = ds.get(variable) diff --git a/xclim/testing/tests/data/eastern_canada.json b/xclim/testing/tests/data/eastern_canada.json deleted file mode 100644 index b2e74e08c..000000000 --- a/xclim/testing/tests/data/eastern_canada.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "properties": {}, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - -75.9375, - 53.12040528310657 - ], - [ - -87.1875, - 45.82879925192134 - ], - [ - -72.0703125, - 38.272688535980976 - ], - [ - -59.765625, - 46.07323062540835 - ], - [ - -60.46875, - 56.17002298293205 - ], - [ - -75.9375, - 53.12040528310657 - ] - ] - ] - } - } - ] -} diff --git a/xclim/testing/tests/data/meridian.json b/xclim/testing/tests/data/meridian.json deleted file mode 100644 index 995dff563..000000000 --- a/xclim/testing/tests/data/meridian.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "properties": {}, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 5.8671875, - 57.326521225217064 - ], - [ - -15.468749999999998, - 48.45835188280866 - ], - [ - -16.171875, - 24.84656534821976 - ], - [ - -3.8671874999999996, - 13.581920900545844 - ], - [ - 21.796875, - 25.799891182088334 - ], - [ - 22.8515625, - 52.482780222078226 - ], - [ - 5.8671875, - 57.326521225217064 - ] - ] - ] - } - } - ] -} diff --git a/xclim/testing/tests/data/meridian_multi.json b/xclim/testing/tests/data/meridian_multi.json deleted file mode 100644 index fbefd70bd..000000000 --- a/xclim/testing/tests/data/meridian_multi.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "type": "FeatureCollection", - "features": [ - { - "id": "0", - "type": "Feature", - "properties": {}, - "geometry": { - "type": "MultiPolygon", - "coordinates": [ - [ - [ - [ - -22.148437499999996, - 54.16243396806779 - ], - [ - -29.003906249999996, - 53.85252660044951 - ], - [ - -29.70703125, - 48.22467264956519 - ], - [ - -27.773437499999996, - 42.8115217450979 - ], - [ - -16.34765625, - 47.39834920035926 - ], - [ - -22.148437499999996, - 54.16243396806779 - ] - ] - ], - [ - [ - [ - -23.73046875, - 58.44773280389084 - ], - [ - 10.1953125, - 58.26328705248601 - ], - [ - 8.0859375, - 65.29346780107583 - ], - [ - -21.26953125, - 64.77412531292873 - ], - [ - -23.73046875, - 58.44773280389084 - ] - ] - ] - ] - } - } - ] -} diff --git a/xclim/testing/tests/data/multi_regions.json b/xclim/testing/tests/data/multi_regions.json deleted file mode 100644 index f5d797b13..000000000 --- a/xclim/testing/tests/data/multi_regions.json +++ /dev/null @@ -1,168 +0,0 @@ -{ - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "properties": { - "id": "Québec" - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - -79.8046875, - 63.39152174400882 - ], - [ - -81.9140625, - 48.22467264956519 - ], - [ - -75.234375, - 39.639537564366684 - ], - [ - -62.9296875, - 48.69096039092549 - ], - [ - -48.515625, - 54.57206165565852 - ], - [ - -64.6875, - 63.23362741232569 - ], - [ - -78.046875, - 63.39152174400882 - ], - [ - -79.8046875, - 63.39152174400882 - ] - ] - ] - } - }, - { - "type": "Feature", - "properties": { - "id": "Europe" - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - -18.28125, - 68.52823492039876 - ], - [ - -27.0703125, - 66.37275500247455 - ], - [ - -18.28125, - 61.10078883158897 - ], - [ - -11.25, - 55.57834467218206 - ], - [ - -13.0078125, - 46.800059446787316 - ], - [ - -10.1953125, - 32.84267363195431 - ], - [ - 9.140625, - 37.16031654673677 - ], - [ - 29.53125, - 33.137551192346145 - ], - [ - 32.6953125, - 42.8115217450979 - ], - [ - 43.2421875, - 48.45835188280866 - ], - [ - 38.3203125, - 65.6582745198266 - ], - [ - 39.0234375, - 70.8446726342528 - ], - [ - 16.5234375, - 73.42842364106816 - ], - [ - -10.546875, - 71.07405646336098 - ], - [ - -18.28125, - 68.00757101804004 - ], - [ - -18.28125, - 68.52823492039876 - ] - ] - ] - } - }, - { - "type": "Feature", - "properties": { - "id": "Newfoundland" - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - -46.40625, - 57.70414723434193 - ], - [ - -57.65624999999999, - 54.77534585936447 - ], - [ - -64.6875, - 46.800059446787316 - ], - [ - -58.00781249999999, - 40.713955826286046 - ], - [ - -47.4609375, - 45.583289756006316 - ], - [ - -42.890625, - 53.74871079689897 - ], - [ - -46.40625, - 57.70414723434193 - ] - ] - ] - } - } - ] -} diff --git a/xclim/testing/tests/data/poslons.json b/xclim/testing/tests/data/poslons.json deleted file mode 100644 index a8fd4126a..000000000 --- a/xclim/testing/tests/data/poslons.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "properties": {}, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 9.3603515625, - 53.72271667491848 - ], - [ - 4.8779296875, - 52.482780222078226 - ], - [ - 4.46044921875, - 49.710272582105695 - ], - [ - 7.31689453125, - 47.010225655683485 - ], - [ - 10.810546875, - 48.48748647988415 - ], - [ - 12.392578125, - 51.508742458803326 - ], - [ - 9.38232421875, - 51.781435604431195 - ], - [ - 9.3603515625, - 53.72271667491848 - ] - ] - ] - } - } - ] -} diff --git a/xclim/testing/tests/data/small_geojson.json b/xclim/testing/tests/data/small_geojson.json deleted file mode 100644 index 1d4213589..000000000 --- a/xclim/testing/tests/data/small_geojson.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "properties": {}, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - -73.61148834228516, - 45.50610628959769 - ], - [ - -73.61629486083984, - 45.489742295807275 - ], - [ - -73.58745574951172, - 45.48372492603276 - ], - [ - -73.57646942138672, - 45.502978246693786 - ], - [ - -73.59397888183594, - 45.513324186793014 - ], - [ - -73.61148834228516, - 45.50610628959769 - ] - ] - ] - } - } - ] -} diff --git a/xclim/testing/tests/data/southern_qc_geojson.json b/xclim/testing/tests/data/southern_qc_geojson.json deleted file mode 100644 index 4054c4393..000000000 --- a/xclim/testing/tests/data/southern_qc_geojson.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "properties": {}, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - -74.608154296875, - 46.03510927947334 - ], - [ - -76.541748046875, - 46.49082901981415 - ], - [ - -77.618408203125, - 45.92822950933618 - ], - [ - -76.640625, - 44.84808025602074 - ], - [ - -72.88330078125, - 45.251688256117646 - ], - [ - -72.48779296875, - 46.68713141244413 - ], - [ - -73.93798828125, - 46.99524110694593 - ], - [ - -74.674072265625, - 46.240651955001695 - ], - [ - -74.608154296875, - 46.03510927947334 - ] - ] - ] - } - } - ] -} diff --git a/xclim/testing/tests/test_ensembles.py b/xclim/testing/tests/test_ensembles.py index 0d60dd749..59b41da1c 100644 --- a/xclim/testing/tests/test_ensembles.py +++ b/xclim/testing/tests/test_ensembles.py @@ -27,7 +27,7 @@ from xclim import ensembles from xclim.indices.stats import get_dist -from .conftest import MAIN_TESTDATA_BRANCH +from .conftest import TESTDATA_BRANCH class TestEnsembleStats: @@ -57,7 +57,7 @@ def test_create_ensemble( # Kinda a hack? Alternative is to open and rewrite in a temp folder. files = [ - threadsafe_data_dir / MAIN_TESTDATA_BRANCH / "EnsembleStats" / Path(f).name + threadsafe_data_dir / TESTDATA_BRANCH / "EnsembleStats" / Path(f).name for f in ensemble_dataset_objects["nc_files_simple"] ] ens2 = ensembles.create_ensemble(dict(zip(reals, files))) From c7cfc8d42824a606175cef95249887b98f1e2523 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:56:55 -0500 Subject: [PATCH 2/6] stage tox-conda for eventual integration --- .github/workflows/main.yml | 8 ++++---- environment.yml | 1 + pyproject.toml | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 99287a9ea..7653407a5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,7 @@ jobs: with: python-version: "3.8" - name: Install tox - run: pip install tox~=3.0 + run: pip install tox~=4.0 - name: Run linting suite run: tox -e black @@ -56,7 +56,7 @@ jobs: with: python-version: "3.8" - name: Install tox - run: pip install tox~=3.0 + run: pip install tox~=4.0 - name: Test with tox run: tox -e py38 env: @@ -76,7 +76,7 @@ jobs: with: python-version: "3.9" - name: Install tox - run: pip install tox~=3.0 + run: pip install tox~=4.0 - name: Test with tox run: tox -e py39-upstream-lm3-coverage env: @@ -156,7 +156,7 @@ jobs: with: python-version: "3.8" - name: Install tox - run: pip install tox~=3.0 + run: pip install tox~=4.0 - name: Test with tox run: tox -e py38-lm3-coverage env: diff --git a/environment.yml b/environment.yml index 7af87d903..59bb1df89 100644 --- a/environment.yml +++ b/environment.yml @@ -58,6 +58,7 @@ dependencies: - statsmodels - tokenize-rt - tox +# - tox-conda # Will be added when a tox@v4.0+ compatible plugin is released. - xdoctest - yamale - yamllint diff --git a/pyproject.toml b/pyproject.toml index 0ad4b938b..5d9f9cae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ dev = [ "pytest-xdist[psutil] >=3.2", "tokenize-rt", "tox", + # "tox-conda", # Will be added when a tox@v4.0+ compatible plugin is released. "xdoctest", "yamale", "yamllint", From b69b86744ce643d8ca0768a966825df28feb9c78 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:59:02 -0500 Subject: [PATCH 3/6] upgrade to tox>=4.0, adjust tox configuration to set pytest-xdist and coverage options as needed, adjust pyproject.toml to not use pytest-xdist/report coverage by default --- .github/workflows/main.yml | 9 +++------ pyproject.toml | 6 ++---- tox.ini | 27 ++++++++++++++++----------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7653407a5..d7d402cbc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,7 +20,7 @@ on: - submitted env: - MAIN_TESTDATA_BRANCH: main + XCLIM_TESTDATA_BRANCH: main jobs: black: @@ -115,8 +115,9 @@ jobs: python -m pip install --no-user . - name: Test run: | + pip list pip check - pytest xclim --durations=10 + pytest xclim --num-workers=logical --durations=10 --cov=xclim --cov-report=term-missing env: CONDA_EXE: mamba GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -184,10 +185,6 @@ jobs: run: pylint --rcfile=pylintrc --disable=import-error --exit-zero xclim - name: Test with tox run: tox -e notebooks_doctests - - name: Report coverage - run: | - pip install --upgrade coveralls - coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-notebooks_doctests-py38 diff --git a/pyproject.toml b/pyproject.toml index 5d9f9cae8..c1bdfa15a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -166,10 +166,8 @@ ignore_missing_imports = true addopts = [ "--verbose", "--color=yes", - "--cov=xclim", - "--cov-report=term-missing", - "--numprocesses=auto", - "--maxprocesses=6", + "--numprocesses=0", + "--maxprocesses=8", "--dist=worksteal" ] norecursedirs = ["docs/notebooks/*"] diff --git a/tox.ini b/tox.ini index ce348df9c..d5e8df350 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,15 @@ [tox] +min_version = 4.0 env_list = black docs - notebooks_doctests + notebooks_doctests{-coverage,} ; opt-slow py38-lm3 py39-upstream-doctest py310-slow -isolated_build = True requires = pip >= 21.0 - tox ~=3.0 opts = -v [testenv:black] @@ -35,6 +34,7 @@ commands = blackdoc --check docs isort --check xclim yamllint --config-file=.yamllint.yaml xclim +commands_post = [testenv:docs] description = Build the documentation with makefile under {basepython} @@ -44,6 +44,7 @@ setenv = READTHEDOCS = 1 commands = make docs +commands_post = allowlist_externals = env make @@ -55,11 +56,11 @@ allowlist_externals = ;conda_env = environment-dev.yml ;extras = -[testenv:notebooks_doctests] +[testenv:notebooks_doctests{-coverage,}] description = Run notebooks and doctests with pytest under {basepython} commands = - pytest --no-cov --nbval --dist=loadscope docs/notebooks --durations=10 --ignore=docs/notebooks/example.ipynb - pytest --rootdir xclim/testing/tests/ --xdoctest xclim --ignore=xclim/testing/tests/ --durations=10 + pytest --no-cov --nbval --dist=loadscope docs/notebooks --ignore=docs/notebooks/example.ipynb + pytest --rootdir xclim/testing/tests/ --xdoctest xclim --ignore=xclim/testing/tests/ # Requires tox-conda compatible with tox@v4.0 ;[testenv:opt-{slow,not_slow}] @@ -76,6 +77,8 @@ commands = description = Run tests with pytest under {basepython} setenv = COV_CORE_SOURCE = + PYTEST_ADDOPTS = --numprocesses=logical --durations=10 + coverage: PYTEST_ADDOPTS = --numprocesses=logical --durations=10 --cov=xclim --cov-report=term-missing PYTHONPATH = {toxinidir} passenv = CI @@ -83,7 +86,7 @@ passenv = COVERALLS_* GITHUB_* LD_LIBRARY_PATH - MAIN_TESTDATA_BRANCH + XCLIM_* extras = dev deps = coverage: coveralls @@ -91,12 +94,14 @@ deps = upstream: -rrequirements_upstream.txt install_command = python -m pip install --no-user {opts} {packages} download = True -commands = +commands_pre = pip list pip check - doctest: pytest --no-cov --rootdir xclim/testing/tests/ --xdoctest xclim --ignore=xclim/testing/tests/ --durations=10 - !slow: pytest xclim -m "not slow" --durations=10 - slow: pytest xclim --durations=10 +commands = + doctest: pytest --no-cov --rootdir xclim/testing/tests/ --xdoctest xclim --ignore=xclim/testing/tests/ + !slow: pytest xclim -m "not slow" + slow: pytest xclim +commands_post = coverage: - coveralls allowlist_externals = git From 3de2ace792424b8da366d0aadf59aec6a755907f Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 21 Feb 2023 14:11:22 -0500 Subject: [PATCH 4/6] update HISTORY.rst --- HISTORY.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1b19b5b84..21a0bfada 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,18 @@ New indicators * New indices and indicators for determining upwelling radiation (`shortwave_upwelling_radiation_from_net_downwelling` and `longwave_upwelling_radiation_from_net_downwelling`; CF variables `rsus` and `rlus`) from net and downwelling radiation (shortwave: `rss` and `rsds`; longwave: `rls` and `rlds`). (:pull:`1271`). * New indice and indicator (``dryness_index``) for estimating soil humidity classifications for winegrowing regions (based on Riou et al. (1994)). (:issue:`355`, :pull:`1235`). +Breaking changes +^^^^^^^^^^^^^^^^ +* `xclim` testing default behaviours have been changed (:issue:`1295`, :pull:`1297`): + * Running `$ pytest` will no longer use `pytest-xdist` distributed testing be default (can be set with ``-n auto|logical|#``. Coverage is also no longer gathered/reported by default. + * Running `$ tox` will now set `pytest-xdist` to use ``-n logical`` processes (with a max of 10). + * Default behaviour for testing is to no longer always fetch `xclim-testdata`. If testdata is found in ``$HOME/.xclim_testing_data``, files will be copied to individual processes, otherwise, will be fetched as needed. +* Environment variables evaluated when running pytest have been changed (:issue:`1295`, :pull:`1297`): + * For testing against specific branches of `xclim-testdata`: ``MAIN_TESTDATA_BRANCH`` -> ``XCLIM_TESTDATA_BRANCH`` + * The option to skip fetching of testdata (``SKIP_TEST_DATA``) has been removed + * A new environment variable (``XCLIM_PREFETCH_TESTING_DATA``) is now available to gather `xclim-testdata` before running test ensemble (default: `False`). + * Environment variables are now passed to `tox` on execution. + Bug fixes ^^^^^^^^^ * ``build_indicator_module_from_yaml`` now accepts a ``reload`` argument. When re-building a module that already exists, ``reload=True`` removes all previous indicator before creating the new ones. (:issue:`1192`, :pull:`1284`). @@ -27,7 +39,8 @@ Internal changes * Coveralls GitHub Action removed as it did not support ``pyproject.toml``-based configurations. (:pull:`1278`). * Add a remark about how `xclim`'s CFFWIS is different from the original 1982 implementation. (:issue:`1104`, :pull:`1284`). * Update CI runs to use Python3.9 when examining upstream dependencies. Replace `setup-conda` action with `provision-with-micromamba` action. (:pull:`1286`). -* Update CI runs to always use `tox~=3.0` and the `latest` Ubuntu images (now `v22.04`). (:pull:`1288`). +* ~Update CI runs to always use `tox~=3.0` and the `latest` Ubuntu images (now `v22.04`). (:pull:`1288`).~ + * CI and tox defaults now use `tox~4.0` and the `latest` Ubuntu images. (:pull:`1297`). * `SBCK` installation command now points to the official development repository. (:pull:`1288`). * Some references in the BibTeX were updated to point to better resources. (:pull:`1288`). * Add a GitHub CI workflow for performing dependency security review scanning. (:pull:`1287`). From 18cbf0a4f8cee2a17a058101c515b9f139e5175d Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 24 Feb 2023 12:07:40 -0500 Subject: [PATCH 5/6] update HISTORY.rst --- HISTORY.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 95b18fe8a..4d89b9263 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -39,8 +39,7 @@ Internal changes * Coveralls GitHub Action removed as it did not support ``pyproject.toml``-based configurations. (:pull:`1278`). * Add a remark about how `xclim`'s CFFWIS is different from the original 1982 implementation. (:issue:`1104`, :pull:`1284`). * Update CI runs to use Python3.9 when examining upstream dependencies. Replace `setup-conda` action with `provision-with-micromamba` action. (:pull:`1286`). -* ~Update CI runs to always use `tox~=3.0` and the `latest` Ubuntu images (now `v22.04`). (:pull:`1288`).~ - * CI and tox defaults now use `tox~4.0` and the `latest` Ubuntu images. (:pull:`1297`). +* Update CI runs to always use `tox~=4.0` and the `latest` virtual machine images (now `ubuntu-22.04`). (:pull:`1288`, :pull:`1297`). * `SBCK` installation command now points to the official development repository. (:pull:`1288`). * Some references in the BibTeX were updated to point to better resources. (:pull:`1288`). * Add a GitHub CI workflow for performing dependency security review scanning. (:pull:`1287`). From 3e43ce928a4f755faeab35b7a35045ed8f67c18b Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 24 Feb 2023 12:09:25 -0500 Subject: [PATCH 6/6] fix call --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d7d402cbc..43c1faed4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -117,7 +117,7 @@ jobs: run: | pip list pip check - pytest xclim --num-workers=logical --durations=10 --cov=xclim --cov-report=term-missing + pytest xclim --numprocesses=logical --durations=10 --cov=xclim --cov-report=term-missing env: CONDA_EXE: mamba GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}