From f8def5a0bd0823fde56f7ff0d7f1d1b298d5b683 Mon Sep 17 00:00:00 2001 From: James McCreight Date: Tue, 18 Jul 2023 14:37:07 -0600 Subject: [PATCH 1/8] update version for develop --- pywatershed/version.py | 4 ++-- version.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pywatershed/version.py b/pywatershed/version.py index dfcbc3cd..2be6b2f2 100644 --- a/pywatershed/version.py +++ b/pywatershed/version.py @@ -1,6 +1,6 @@ -# pywatershed version file automatically created using update_version.py on July 18, 2023 16:50:45 +# pywatershed version file automatically created using update_version.py on July 18, 2023 14:36:23 -__version__ = "0.2.0" +__version__ = "0.3.0.dev0" __pakname__ = "pywatershed" author_dict = { diff --git a/version.txt b/version.txt index 341cf11f..2934a2fc 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.2.0 \ No newline at end of file +0.3.0.dev0 \ No newline at end of file From d89b0738be1706a504a9c8344c0fbba43d3ff50e Mon Sep 17 00:00:00 2001 From: James McCreight Date: Tue, 18 Jul 2023 14:40:06 -0600 Subject: [PATCH 2/8] update whats-new post release --- doc/whats-new.rst | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d1a5f4ef..d8288d32 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -12,9 +12,42 @@ What's New np.random.seed(123456) +.. _whats-new.0.3.0: + +v0.3.0 (unreleased) +--------------------- + +New features +~~~~~~~~~~~~ + + +Breaking changes +~~~~~~~~~~~~~~~~ + + +Deprecations +~~~~~~~~~~~~ + + +Performance +~~~~~~~~~~~ + + +Bug fixes +~~~~~~~~~ + + +Documentation +~~~~~~~~~~~~~ + + +Internal changes +~~~~~~~~~~~~~~~~ + + .. _whats-new.0.2.0: -v0.2.0 (12 July 2023) +v0.2.0 (18 July 2023) --------------------- New features From b3dedc9bb92a9d5713a386c2b0e86036ba026a6c Mon Sep 17 00:00:00 2001 From: James McCreight Date: Wed, 19 Jul 2023 11:34:27 -0600 Subject: [PATCH 3/8] fixes for 0.2.1 --- doc/whats-new.rst | 32 ++++++---------------------- examples/02_prms_legacy_models.ipynb | 6 ++++-- pyproject.toml | 2 +- 3 files changed, 11 insertions(+), 29 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d8288d32..7a32dad1 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -12,37 +12,17 @@ What's New np.random.seed(123456) -.. _whats-new.0.3.0: +.. _whats-new.0.2.1: -v0.3.0 (unreleased) +v0.2.1 (19 July 2023) --------------------- -New features -~~~~~~~~~~~~ - - -Breaking changes -~~~~~~~~~~~~~~~~ - - -Deprecations -~~~~~~~~~~~~ - - -Performance -~~~~~~~~~~~ - - Bug fixes ~~~~~~~~~ - - -Documentation -~~~~~~~~~~~~~ - - -Internal changes -~~~~~~~~~~~~~~~~ +- Package data was not properly installed. + (:pull:`219`) By `James McCreight `_. +- Small addition to notebook 02 + (:pull:`219`) By `James McCreight `_. .. _whats-new.0.2.0: diff --git a/examples/02_prms_legacy_models.ipynb b/examples/02_prms_legacy_models.ipynb index bc137515..635effc1 100644 --- a/examples/02_prms_legacy_models.ipynb +++ b/examples/02_prms_legacy_models.ipynb @@ -410,8 +410,10 @@ " \"input_dir\": run_dir,\n", " \"budget_type\": \"warn\",\n", " \"calc_method\": \"numba\",\n", - " # \"netcdf_output_dir\": nb_output_dir / \"nhm\",\n", - "}" + " \"netcdf_output_dir\": nb_output_dir / \"nhm_submodel\",\n", + "}\n", + "\n", + "control.options[\"netcdf_output_var_names\"] = pws.PRMSChannel.get_variables()" ] }, { diff --git a/pyproject.toml b/pyproject.toml index ae3b248a..19d5394e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,7 +110,7 @@ include = ["pywatershed", "pywatershed.*"] [tool.setuptools.package-data] "pywatershed.static.metadata" = ["*.yaml"] -"pywatershed.data" = ["*.nc"] +"pywatershed.data.drb_2yr" = ["*"] [tool.black] line-length = 79 From e608bfe736945f738d2367b6e24265eae825777c Mon Sep 17 00:00:00 2001 From: James McCreight Date: Wed, 19 Jul 2023 15:32:35 -0600 Subject: [PATCH 4/8] env and PR template details --- .github/PULL_REQUEST_TEMPLATE.md | 4 +++- environment_w_jupyter.yml | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 626e4f80..4fb7c8b3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,8 @@ - [ ] Closes #xxxx -- [ ] Tests and/or performance benchmarks added +- [ ] Tests added +- [ ] Performance benchmarks added +- [ ] Performance regression benchmarks run - [ ] User visible changes (including notable bug fixes) are documented in `whats-new.rst` - [ ] New functions/methods are listed in `api.rst` or it's sub rsts? \ No newline at end of file diff --git a/environment_w_jupyter.yml b/environment_w_jupyter.yml index f02cd89f..8c25abac 100644 --- a/environment_w_jupyter.yml +++ b/environment_w_jupyter.yml @@ -17,7 +17,6 @@ dependencies: - ipython - ipywidgets - jupyter - - jupyter_contrib_nbextensions - jupyterlab - nb_conda_kernels - nbconvert @@ -45,11 +44,13 @@ dependencies: - tqdm - xarray>=2023.05.0 - pip: + - asv - click != 8.1.0 - black - isort - flake8 - jupyter_black + - jupyter_contrib_nbextensions - pylint - git+https://github.com/modflowpy/flopy.git - modflow-devtools From 927f8dac03a6fa97456eee97ea43caade4781637 Mon Sep 17 00:00:00 2001 From: James McCreight Date: Wed, 19 Jul 2023 21:14:13 -0600 Subject: [PATCH 5/8] touch up envs, drop jupyter_contrib_nbextensions --- environment.yml | 12 ++++++------ environment_w_jupyter.yml | 11 +++++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/environment.yml b/environment.yml index 38d01ecc..4de85c24 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: pywatershed +name: pws channels: - conda-forge - nodefaults @@ -11,12 +11,11 @@ dependencies: - filelock - geopandas - geoviews + - git - holoviews - hvplot - - git - ipython - ipywidgets - - jupyter_contrib_nbextensions - nb_conda_kernels - nbconvert - netCDF4 @@ -43,11 +42,12 @@ dependencies: - tqdm - xarray>=2023.05.0 - pip: - - click != 8.1.0 + - asv - black + - click != 8.1.0 - isort - flake8 - - jupyter_black - - pylint - git+https://github.com/modflowpy/flopy.git + - jupyter_black - modflow-devtools + - pylint diff --git a/environment_w_jupyter.yml b/environment_w_jupyter.yml index 8c25abac..025abe65 100644 --- a/environment_w_jupyter.yml +++ b/environment_w_jupyter.yml @@ -1,4 +1,4 @@ -name: pywatershed +name: pws channels: - conda-forge - nodefaults @@ -11,9 +11,9 @@ dependencies: - filelock - geopandas - geoviews + - git - holoviews - hvplot - - git - ipython - ipywidgets - jupyter @@ -45,12 +45,11 @@ dependencies: - xarray>=2023.05.0 - pip: - asv - - click != 8.1.0 - black + - click != 8.1.0 - isort - flake8 - - jupyter_black - - jupyter_contrib_nbextensions - - pylint - git+https://github.com/modflowpy/flopy.git + - jupyter_black - modflow-devtools + - pylint From 107d18ab633431b77f0d128a177bb4310a4fc8f5 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Thu, 20 Jul 2023 15:06:06 -0400 Subject: [PATCH 6/8] ci: handle empty changelog in release automation --- .github/workflows/release.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5ebf10d6..a255baf3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -193,7 +193,14 @@ jobs: run: | version=$(cat version.txt | xargs) title="pywatershed $version" - notes=$(cat "changelog/CHANGELOG_$version.md" | grep -v "### Version $version") + + # don't fail if the changelog is empty + if [[ -s "changelog/CHANGELOG_$version.md" ]]; then + notes=$(cat "changelog/CHANGELOG_$version.md" | grep -v "### Version $version") + else + notes="No changes found, are recent commits conventional?" + fi + gh release create "$version" \ --target main \ --title "$title" \ From 0249c038c89b98b1460dd3b1c9b3619c7093b1a3 Mon Sep 17 00:00:00 2001 From: James McCreight Date: Thu, 20 Jul 2023 17:24:19 -0600 Subject: [PATCH 7/8] polishes --- .github/RELEASE.md | 24 +++--- README.md | 109 +++++++++++++++++----------- asv_benchmarks/benchmarks/prms.py | 21 +++++- asv_benchmarks/pws_refs_for_asv.txt | 3 + 4 files changed, 100 insertions(+), 57 deletions(-) create mode 100644 asv_benchmarks/pws_refs_for_asv.txt diff --git a/.github/RELEASE.md b/.github/RELEASE.md index 455932f9..3021bddc 100644 --- a/.github/RELEASE.md +++ b/.github/RELEASE.md @@ -47,19 +47,21 @@ To release a new version: remote: `git push -u upstream vx.y.z`. This starts a job to: - Check out the release branch Update version number in `version.txt` and - - `pywatershed/version.py` to match the version in the branch name Build and - - check the Python package Generate a changelog since the last release - - Prepend the changelog to the cumulative `HISTORY.md` Upload the package - - and changelog as artifacts Draft a PR against `main` with the updated - - version files and cumulative changelog. The cumulative `HISTORY.md` is - - version-controlled, release changelogs are not. + `pywatershed/version.py` to match the version in the branch name + - Build and check the Python package Generate a changelog since the last + release + - Prepend the changelog to the cumulative `HISTORY.md`. The cumulative + `HISTORY.md` is version-controlled, release changelogs are not. + - Upload the package and changelog as artifacts Draft a PR against `main` + with the updated version files and cumulative changelog. 1. On all platforms, pull the release from upstream and perform ASV performance - benchmarks against previous release , e.g., ``` asv continuous --verbose - --show-stderr --factor 1.3 previous_release this_release ``` Collect - performance reports from various machines into a single report and use `asv - publish` to generate the static webpages to be included with the release as - artifacts in that step below. + benchmarks against previous release , e.g., ```asv run --verbose + --show-stderr HASHFILE:pws_refs_for_asv.txt``` after editing the file to + contain the previous and current release. Collect performance results from + various machines into a single report and use `asv publish` to generate + the static webpages to be included with the release as artifacts in that + step below. 1. Inspect the package and changelog. If they look good, merge the PR to `main`. diff --git a/README.md b/README.md index 18df5ec5..75872bbb 100644 --- a/README.md +++ b/README.md @@ -38,56 +38,80 @@ The following motivations are taken from our [AGU poster from December which provides additional details on motivations, project status, and current directions of this project as of approximately January 2023. -Goals of the USGS Enterprise Capacity (EC) project include: * A sustainable -integrated, hydrologic modeling framework for the U.S. Geological Survey (USGS) -* Interoperable modeling across the USGS, partner agencies, and academia - -Goals for EC Watershed Modeling: * Couple the Precipitation-Runoff Modeling -System (PRMS, e.g. Regan et al, 2018)  with MODFLOW 6 (MF6, e.g. Langevin et al, -2017) in a sustainable way * Redesign PRMS to be more modern and flexible * -Prioritize process representations in the current National Hydrological Model -(NHM) based on PRMS 5.2.1 - -Prototype an EC watershed model: "pywatershed" * Redesign PRMS quickly in python -* Couple to MF6 via BMI/XMI interface (Hughes et al, 2021; Hutton et al, 2020) * -Establish a prototyping ground for EC codes that couples to the compiled -framework: low cost proof of concepts (at the price of potentially less -computational performance) * Enable process representation hypothesis testing * -Use cutting-edge techniques and technologies to improve models * Machine -learning, automatic differentiation * Address challenges of modeling across -space and time scales * Transition prototype watershed model to compiled EC code +Goals of the USGS Enterprise Capacity (EC) project include: -## Installation + * A sustainable integrated, hydrologic modeling framework for the U.S. + Geological Survey (USGS) + * Interoperable modeling across the USGS, partner agencies, and academia + +Goals for EC Watershed Modeling: -To install the software you will need Python 3.9 or 3.10. + * Couple the Precipitation-Runoff Modeling System (PRMS, e.g. Regan et al, + 2018)  with MODFLOW 6 (MF6, e.g. Langevin et al, 2017) in a sustainable + way + * Redesign PRMS to be more modern and flexible + * Prioritize process representations in the current National Hydrological + Model (NHM) based on PRMS 5.2.1 -We currently recommend dependencies be installed with -[Mamba](https://mamba.readthedocs.io/en/latest/) which will be much faster than -Ananconda (but the conda command can also be used). An environment containing -all core and optional dependencies can be created from the project root with: +Prototype an EC watershed model: "pywatershed" -```mamba env create -f environment_w_jupyter.yml``` + * Redesign PRMS quickly in python + * Couple to MF6 via BMI/XMI interface (Hughes et al, 2021; Hutton et al, 2020) + * Establish a prototyping ground for EC codes that couples to the compiled + framework: low cost proof of concepts (at the price of potentially less + computational performance) * Enable process representation hypothesis testing + * Use cutting-edge techniques and technologies to improve models + * Machine learning, automatic differentiation + * Address challenges of modeling across space and time scales + * Transition prototype watershed model to compiled EC code -(The environment `environment.yml` does not contain jupyter or jupyterlab -in order to not interfere with installation in WholeTale, see Example -Notebooks seection below.) + +## Installation + +`pywatershed` uses Python 3.9 or 3.10. The `pywatershed` package is [available on -PyPI](https://pypi.org/project/pywatershed/). At the moment, the installation -may not be reliable on all platforms and we are working to fix this. +PyPI](https://pypi.org/project/pywatershed/) but installation of dependencies +may not be reliable on all platforms. -Using PyPI (with the above caveat), `pywatershed` can be installed with: +We recommend dependencies be first installed with +[Mamba](https://mamba.readthedocs.io/en/latest/). This will be much faster than +Ananconda (but the conda command could also be used). We hope to provide +`pywatershed` installation with all its dependencies on conda-forge in the +near future. -``` pip install pywatershed ``` +If you wish to use the stable release, you will use `main` in place of +`` in the following commands. If you want to follow developemnt, you'll +use `develop` instead. -A number of extra dependencies are needed to run the example notebooks. These -can be installed with pip with +Without using `git` (directly), you may: +``` +curl -L -O https://raw.githubusercontent.com/EC-USGS/pywatershed//environment_w_jupyter.yml +mamba env create -f environment_w_jupyter.yml +conda activate pws +pip install git+https://github.com/EC-USGS/pywatershed.git@ +``` -``` pip install "pywatershed[optional]" ``` +Or to use `git` and to be able to develop: -These installation steps are suitable for `pywatershed` end users. See the -[developer documentation](./DEVELOPER.md) for detailed instructions on -configuring a development environment. +``` +git clone https://github.com/EC-USGS/pywatershed.git +cd pywatershed +mamba env create -f environment_w_jupyter.yml +activate pws +pip install -e . +``` + +(If you want to name the environment other than the default `pws`, use the +command +`mamba env update --name your_env_name --file environment_w_jupyter.yml --prune` +you will also need to activate this environment by name.) + + +We install the `environment_w_jupyter.yml` to provide all known dependencies +including those for running the eample notebooks. (The `environment.yml` +does not contain jupyter or jupyterlab because this interferes with installation +on WholeTale, see Example Notebooks seection below.) ## Contributing @@ -116,10 +140,11 @@ to register. There are containers for both the `main` and `develop` branches. [![WholeTale](https://raw.githubusercontent.com/whole-tale/wt-design-docs/master/badges/wholetale-explore.svg)](https://dashboard.wholetale.org) -* [WholeTale container for latest release (main - branch)](https://dashboard.wholetale.org/run/64ae29e8a887f48b9f173678?tab=metadata) -* [WholeTale container for develop - branch](https://dashboard.wholetale.org/run/64ae25c3a887f48b9f1735c8?tab=metadata) + + * [WholeTale container for latest release (main + branch)](https://dashboard.wholetale.org/run/64ae29e8a887f48b9f173678?tab=metadata) + * [WholeTale container for develop + branch](https://dashboard.wholetale.org/run/64ae25c3a887f48b9f1735c8?tab=metadata) WholeTale will give you a jupyter-lab running in the root of this repository. You can navigate to `examples/` and then open and run the notebooks diff --git a/asv_benchmarks/benchmarks/prms.py b/asv_benchmarks/benchmarks/prms.py index 1209338c..335ed1c6 100644 --- a/asv_benchmarks/benchmarks/prms.py +++ b/asv_benchmarks/benchmarks/prms.py @@ -85,17 +85,17 @@ def setup(self, *args): else: self.params = pws.PrmsParameters.load(self.parameter_file) - # backwards compatability + # backwards compatability pre 0.2.0 try: + self.ge_v0_2_0 = False self.control = pws.Control.load( self.control_file, params=self.params ) - self.ge_v0_2_0 = False except: - self.control = pws.Control.load(self.control_file) self.ge_v0_2_0 = True - self.control.edit_n_time_steps(n_time_steps) + if hasattr(self, "control"): + del self.control # setup input_dir with symlinked prms inputs and outputs self.domain_dir = pl.Path(f"PRMSModels_{self.domain}") @@ -119,17 +119,28 @@ def model_setup_run( processes: tuple = None, write_output: Union[bool, Literal["separate", "together"]] = None, ): + # seem to need to load control inside the model setup run bc + # results are strange/inconsistent + if self.ge_v0_2_0: + self.control = pws.Control.load(self.control_file) self.control.options["input_dir"] = self.tag_input_dir self.control.options["budget_type"] = "warn" self.control.options["calc_method"] = "numba" + self.control.edit_n_time_steps(n_time_steps) model = pws.Model( self.processes, control=self.control, parameters=self.params, ) + else: + self.control = pws.Control.load( + self.control_file, params=self.params + ) + self.control.edit_n_time_steps(n_time_steps) + model = pws.Model( *self.processes, control=self.control, @@ -143,6 +154,8 @@ def model_setup_run( self.tag_dir, separate_files=(write_output == "separate") ) model.run(finalize=True) + del model + del self.control @parameterized( ["domain", "processes", "output"], diff --git a/asv_benchmarks/pws_refs_for_asv.txt b/asv_benchmarks/pws_refs_for_asv.txt new file mode 100644 index 00000000..f20c0288 --- /dev/null +++ b/asv_benchmarks/pws_refs_for_asv.txt @@ -0,0 +1,3 @@ +0.1.1 +0.2.0 + From 85d96913d608ecf1ab560d1261ed220650009291 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 02:12:15 +0000 Subject: [PATCH 8/8] ci(release): set version to 0.2.1, update changelog --- pywatershed/version.py | 4 ++-- version.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pywatershed/version.py b/pywatershed/version.py index 2be6b2f2..db9143e9 100644 --- a/pywatershed/version.py +++ b/pywatershed/version.py @@ -1,6 +1,6 @@ -# pywatershed version file automatically created using update_version.py on July 18, 2023 14:36:23 +# pywatershed version file automatically created using update_version.py on July 21, 2023 02:12:07 -__version__ = "0.3.0.dev0" +__version__ = "0.2.1" __pakname__ = "pywatershed" author_dict = { diff --git a/version.txt b/version.txt index 2934a2fc..7dff5b89 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.3.0.dev0 \ No newline at end of file +0.2.1 \ No newline at end of file