From a56d76fa211b46b3e261ae2fa3fdd209c759ae6e Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 13:23:16 -0700 Subject: [PATCH 01/20] Simplify conda environments and loosen constraints - Loosening constraints improves upgradeability of the environments with a tradeoff in determinism --- .readthedocs.yml | 8 ++--- conda-env/ci.yml | 19 ++++++----- conda-env/dev.yml | 66 ++++++++++++++++++--------------------- conda-env/readthedocs.yml | 41 ------------------------ 4 files changed, 43 insertions(+), 91 deletions(-) delete mode 100644 conda-env/readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml index c33dee3f..3be49e8e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -6,9 +6,9 @@ version: 2 build: - os: "ubuntu-20.04" - tools: - python: "mambaforge-4.10" + os: "ubuntu-20.04" + tools: + python: "mambaforge-4.10" # Build documentation in the docs/ directory with Sphinx sphinx: @@ -19,4 +19,4 @@ formats: - pdf conda: - environment: conda-env/readthedocs.yml + environment: conda-env/dev.yml diff --git a/conda-env/ci.yml b/conda-env/ci.yml index 61c71c84..49175b6f 100644 --- a/conda-env/ci.yml +++ b/conda-env/ci.yml @@ -1,13 +1,12 @@ -# Conda xcdat CI/CD environment (used in GH Actions) +# Conda xcdat CI/CD environment (used in GH Actions). name: xcdat_ci channels: - conda-forge - defaults dependencies: - # ================== - # Base - # ================== - - python >=3.8 + # Base - required for building the package. + # ========================================= + - python >=3.9 - pip - cf_xarray - cftime @@ -15,14 +14,14 @@ dependencies: - esmpy - lxml - netcdf4 - - numpy + - numpy >=1.23.0 # This version of numpy includes support for Python 3.11. - pandas - python-dateutil - - xarray - # Constrained because 0.6.3 breaks with import ESMF - # Source: https://github.com/pangeo-data/xESMF/issues/212 - - xesmf >0.6.3 + - xarray >=2022.02.0 # This version of Xarray drops support for Python 3.9. - xgcm + # Optional - enables additional features. + # ========================================= + - xesmf >=0.7.0 # Constrained because https://github.com/pangeo-data/xESMF/issues/212. # Quality Assurance # ================== - types-python-dateutil diff --git a/conda-env/dev.yml b/conda-env/dev.yml index c09b316f..cf6ff8cd 100644 --- a/conda-env/dev.yml +++ b/conda-env/dev.yml @@ -4,40 +4,36 @@ channels: - conda-forge - defaults dependencies: - # ================== - # Base - # ================== - # NOTE: If versions are updated, also `additional_dependencies` list for mypy in `.pre-commit-config.yaml` - - python=3.10.10 - - pip=23.0.1 - - cf_xarray=0.8.0 - - cftime=1.6.2 - - dask=2023.3.2 - - lxml=4.9.2 - - netcdf4=1.6.3 - - numpy=1.23.5 - - pandas=1.5.3 - - python-dateutil=2.8.2 - - xarray=2023.4.2 - - xgcm=0.8.0 - # ================== - # Optional - # ================== - - xesmf=0.7.0 - - matplotlib-base=3.7.1 + # Base - required for building the package. + # ========================================= + - python >=3.9 + - pip + - cf_xarray + - cftime + - dask + - esmpy + - lxml + - netcdf4 + - numpy >=1.23.0 # This version of numpy includes support for Python 3.11. + - pandas + - python-dateutil + - xarray >=2022.02.0 # This version of Xarray drops support for Python 3.9. + - xgcm + # Optional - enables additional features. + # ========================================= + - xesmf >=0.7.0 # Constrained because https://github.com/pangeo-data/xESMF/issues/212. + - matplotlib-base >=3.7.0 - nc-time-axis=1.4.1 - # ================== # Documentation # ================== - - sphinx=5.3.0 - - sphinx-autosummary-accessors=2022.4.0 - - sphinx-book-theme=1.0.1 - - sphinx-copybutton=0.5.1 - - nbsphinx=0.9.1 - - pandoc=3.1.1 - - ipython=8.11.0 # Required for nbsphinx syntax highlighting + - sphinx >=5.0.0 + - sphinx-autosummary-accessors + - sphinx-book-theme + - sphinx-copybutton + - nbsphinx + - pandoc + - ipython >=8.0.0 # Required for nbsphinx syntax highlighting - gsw-xarray=0.3.0 # Required for vertical regridding example - # ================== # Quality Assurance # ================== # NOTE: If versions are updated, also update 'rev' in `.pre-commit.config.yaml` @@ -48,14 +44,12 @@ dependencies: - mypy=1.1.1 - pre-commit=3.2.0 - types-python-dateutil=2.8.19 - # ================== # Testing # ================== - - pytest=7.2.2 - - pytest-cov=4.0.0 - # ================== + - pytest + - pytest-cov # Developer Tools # ================== - - jupyterlab=3.6.2 - - tbump=6.9.0 + - jupyterlab + - tbump prefix: /opt/miniconda3/envs/xcdat_dev diff --git a/conda-env/readthedocs.yml b/conda-env/readthedocs.yml deleted file mode 100644 index 17bdcb5e..00000000 --- a/conda-env/readthedocs.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: xcdat_rtd -channels: - - conda-forge - - defaults -dependencies: - # ================== - # Base - # ================== - # NOTE: If versions are updated, also `additional_dependencies` list for mypy in `.pre-commit-config.yaml` - - python=3.10.10 - - pip=23.0.1 - - cf_xarray=0.8.0 - - cftime=1.6.2 - - dask=2023.3.2 - - lxml=4.9.2 - - netcdf4=1.6.3 - - numpy=1.23.5 - - pandas=1.5.3 - - python-dateutil=2.8.2 - - xarray=2023.4.2 - - xgcm=0.8.0 - # ================== - # Optional - # ================== - - xesmf=0.7.0 - - matplotlib-base=3.7.1 - - nc-time-axis=1.4.1 - # ================== - # Documentation - # ================== - - sphinx=5.3.0 - - sphinx-autosummary-accessors=2022.4.0 - - sphinx-book-theme=1.0.1 - - sphinx-copybutton=0.5.1 - - nbsphinx=0.9.1 - - pandoc=3.1.1 - - ipython=8.11.0 # Required for nbsphinx syntax highlighting - # Quality Assurance - # ================== - - types-python-dateutil=2.8.19 -prefix: /opt/miniconda3/envs/xcdat_rtd From f2a7e9cc89b3fac0600047e02905a79b2ec14811 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 14:45:01 -0700 Subject: [PATCH 02/20] Move configs from setup.cfg to pyproject.toml - Bump QA tool versions - Loosen mypy additional dependencies constraints with pre-commit --- .pre-commit-config.yaml | 16 ++++++++-------- conda-env/dev.yml | 8 ++++---- pyproject.toml | 25 +++++++++++++++++++++++++ setup.cfg | 40 +--------------------------------------- 4 files changed, 38 insertions(+), 51 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cddf2200..ca94815a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: check-yaml - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.3.0 hooks: - id: black @@ -30,15 +30,15 @@ repos: additional_dependencies: [flake8-isort] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 + rev: v1.4.0 hooks: - id: mypy - args: ["--config=setup.cfg"] + args: ["--config=pyproject.toml"] additional_dependencies: [ - dask==2023.3.2, - numpy==1.23.5, - pandas==1.5.3, - xarray==2023.3.0, - types-python-dateutil==2.8.19, + dask, + numpy>=1.23.0, + pandas, + xarray>=2023.2.0, + types-python-dateutil, ] diff --git a/conda-env/dev.yml b/conda-env/dev.yml index cf6ff8cd..022ab178 100644 --- a/conda-env/dev.yml +++ b/conda-env/dev.yml @@ -36,14 +36,14 @@ dependencies: - gsw-xarray=0.3.0 # Required for vertical regridding example # Quality Assurance # ================== - # NOTE: If versions are updated, also update 'rev' in `.pre-commit.config.yaml` - - black=23.1.0 + - types-python-dateutil + # NOTE: If the tools below are updated, also update their 'rev' in `.pre-commit.config.yaml` + - black=23.3.0 - flake8=6.0.0 - flake8-isort=6.0.0 - isort=5.12.0 - - mypy=1.1.1 + - mypy=1.4.0 - pre-commit=3.2.0 - - types-python-dateutil=2.8.19 # Testing # ================== - pytest diff --git a/pyproject.toml b/pyproject.toml index 7c50f1a5..116e2f52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,3 +21,28 @@ exclude = ''' )/ ) ''' + +[tool.isort] +# Docs: https://pycqa.github.io/isort/docs/configuration/options.html#example-pyprojecttoml_4 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +line_length = 88 + +[tool.pytest.ini_options] +# Docs: https://docs.pytest.org/en/7.2.x/reference/customize.html#configuration +junit_family = "xunit2" +addopts = "--cov=xcdat --cov-report term --cov-report html:tests_coverage_reports/htmlcov --cov-report xml:tests_coverage_reports/coverage.xml -s" +python_files = ["tests.py", "test_*.py"] +# These markers are defined in `xarray.tests` and must be included to avoid warnings when importing from this module. +markers = ["flaky", "network"] + +[tool.mypy] +# Docs: https://mypy.readthedocs.io/en/stable/config_file.html +python_version = 3.10 +check_untyped_defs = true +ignore_missing_imports = true +warn_unused_ignores = true +warn_redundant_casts = true +warn_unused_configs = true diff --git a/setup.cfg b/setup.cfg index 0c10dbed..c7cd0b32 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,6 @@ [flake8] # https://pep8.readthedocs.io/en/latest/intro.html#error-codes +# flake8 dooes not support pyproject.toml yet: https://github.com/PyCQA/flake8/issues/234 ignore = # whitespace before ‘:’ E203 @@ -23,45 +24,6 @@ exclude = *__init__.py venv -[isort] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 - -[pycodestyle] -max-line-length = 119 -exclude = - .tox - .git - */migrations/* - */static/CACHE/* - docs - node_modules - .idea - .mypy_cache - .pytest_cache - *__init__.py - venv - -[mypy] -python_version = 3.10 -check_untyped_defs = True -ignore_missing_imports = True -warn_unused_ignores = True -warn_redundant_casts = True -warn_unused_configs = True - [aliases] # Define setup.py command aliases here test = pytest - -[tool:pytest] -junit_family=xunit2 -addopts = --cov=xcdat --cov-report term --cov-report html:tests_coverage_reports/htmlcov --cov-report xml:tests_coverage_reports/coverage.xml -s -python_files = tests.py test_*.py -# These markers are defined in `xarray.tests` and must be included to avoid warnings when importing from this module. -markers = - flaky - network From 23d72b892ebafc62b55574a34d07a1e3b2665b40 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 15:08:58 -0700 Subject: [PATCH 03/20] Remove pip and esmpy as dependencies --- conda-env/ci.yml | 2 -- conda-env/dev.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/conda-env/ci.yml b/conda-env/ci.yml index 49175b6f..4551fdcb 100644 --- a/conda-env/ci.yml +++ b/conda-env/ci.yml @@ -7,11 +7,9 @@ dependencies: # Base - required for building the package. # ========================================= - python >=3.9 - - pip - cf_xarray - cftime - dask - - esmpy - lxml - netcdf4 - numpy >=1.23.0 # This version of numpy includes support for Python 3.11. diff --git a/conda-env/dev.yml b/conda-env/dev.yml index 022ab178..1d84b2db 100644 --- a/conda-env/dev.yml +++ b/conda-env/dev.yml @@ -7,11 +7,9 @@ dependencies: # Base - required for building the package. # ========================================= - python >=3.9 - - pip - cf_xarray - cftime - dask - - esmpy - lxml - netcdf4 - numpy >=1.23.0 # This version of numpy includes support for Python 3.11. From ec357d162311757ee24f89c12bb587aa66f1a875 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 15:20:57 -0700 Subject: [PATCH 04/20] Replace conda with mamba instructions --- docs/getting-started.rst | 57 ++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 0a8ffa71..69f06715 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -12,63 +12,62 @@ Prerequisites - We highly recommend visiting the `xarray tutorial`_ and `xarray documentation`_ pages if you aren't familiar with ``xarray``. -2. xCDAT is distributed through conda, which is available through Anaconda and Miniconda. +2. xCDAT is distributed available on Anaconda. We recommend installing `mamba`_, which + is a drop-in replacement of ``conda``. - We recommend following these steps to install Miniconda: + We recommend following these steps to install Mambaforge (Linux example): .. code-block:: console - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh - bash Miniconda3-latest-Linux-x86_64.sh + wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh - Then follow the instructions for installation. To have conda added to - your path you will need to type ``yes`` in response to ``"Do you wish the - installer to initialize Miniconda3 by running conda init?"`` (we recommend - that you do this). Note that this will modify your shell profile (e.g., - ``~/.bashrc``) to add ``conda`` to your path. + bash ./Mambaforge-Linux-x86_64.sh - Note: After installation completes you may need to type ``bash`` to - restart your shell (if you use bash). Alternatively, you can log out and - log back in. + When you see: :: + by running conda init? [yes|no] + [no] >>> yes + + respond with ``yes`` so ``conda`` and ``mamba`` commands are available on + initializing a new bash terminal. + +.. _mamba: https://mamba.readthedocs.io/en/latest/index.html .. _xarray tutorial: https://tutorial.xarray.dev/intro.html .. _xarray documentation: https://docs.xarray.dev/en/stable/getting-started-guide/index.html Installation ------------ -1. Create a conda environment from scratch with ``xcdat`` (`conda create`_) +1. Create a mamba environment from scratch with ``xcdat`` (`mamba create`_) - We recommend using the Conda environment creation procedure to install ``xcdat``. - The advantage with following this approach is that Conda will attempt to resolve - dependencies (e.g. ``python >= 3.8``) for compatibility. + We recommend using the mamba environment creation procedure to install ``xcdat``. + The advantage with following this approach is that mamba will attempt to resolve + dependencies (e.g. ``python >= 3.9``) for compatibility. - To create an ``xcdat`` conda environment with ``xesmf`` (a recommended dependency), - run: + To create an ``xcdat`` environment with ``xesmf`` (a recommended dependency), run: .. code-block:: console - >>> conda create -n -c conda-forge xcdat xesmf - >>> conda activate + >>> mamba create -n -c conda-forge xcdat xesmf + >>> mamba activate Note that ``xesmf`` is an optional dependency, which is required for using ``xesmf`` based horizontal regridding APIs in ``xcdat``. ``xesmf`` is not currently supported - on `osx-arm64`_ or `windows`_ because ``esmpy`` is not yet available on these - platforms. Windows users can try `WSL2`_ as a workaround. + on `windows`_ because ``esmpy`` is not yet available on these platforms. Windows + users can try `WSL2`_ as a workaround. .. _windows: https://github.com/conda-forge/esmf-feedstock/issues/64 -.. _osx-arm64: https://github.com/conda-forge/esmf-feedstock/issues/74 .. _WSL2: https://docs.microsoft.com/en-us/windows/wsl/install -2. Install ``xcdat`` in an existing conda environment (`conda install`_) +2. Install ``xcdat`` in an existing mamba environment (`mamba install`_) - You can also install ``xcdat`` in an existing Conda environment, granted that Conda + You can also install ``xcdat`` in an existing mamba environment, granted that mamba is able to resolve the compatible dependencies. .. code-block:: console - >>> conda activate - >>> conda install -c conda-forge xcdat xesmf + >>> mamba activate + >>> mamba install -c conda-forge xcdat xesmf Note: As above, ``xesmf`` is an optional dependency. @@ -80,5 +79,5 @@ Installation - ``matplotlib``: a library for creating visualizations in Python. - ``cartopy``: an add-on package for ``matplotlib`` and specialized for geospatial data processing. -.. _conda create: https://docs.conda.io/projects/conda/en/latest/commands/create.html?highlight=create -.. _conda install: https://docs.conda.io/projects/conda/en/latest/commands/install.html?highlight=install +.. _mamba create: https://fig.io/manual/mamba/create +.. _mamba install: https://fig.io/manual/mamba/install From 76657635930916c8bc15e1de8f9c6da2f5fd1763 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 15:24:42 -0700 Subject: [PATCH 05/20] Update conda references in contributing guide --- CONTRIBUTING.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 1d79514d..3c8fbe1b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -137,16 +137,16 @@ Local Development Linux :: - $ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh - $ bash ./Miniconda3-latest-Linux-x86_64.sh + $ wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh + $ bash ./Mambaforge-Linux-x86_64.sh Do you wish the installer to initialize Miniconda3 by running conda init? [yes|no] yes MacOS :: - $ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - $ bash ./Miniconda3-latest-MacOSX-x86_64.sh + $ wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-MacOSX-x86_64.sh + $ bash ./Mambaforge-MacOSX-x86_64.sh Do you wish the installer to initialize Miniconda3 by running conda init? [yes|no] yes 2. Fork the ``xcdat`` repo on GitHub. @@ -160,11 +160,11 @@ Local Development 4. Open ``.vscode/xcdat.code-settings`` in VS Code -5. Create and activate Conda development environment:: +5. Create and activate mamba development environment:: $ cd xcdat - $ conda env create -f conda-env/dev.yml - $ conda activate xcdat_dev + $ mamba env create -f conda-env/dev.yml + $ mamba activate xcdat_dev 6. Set VS Code Python interpretor to ``xcdat_dev`` From 7b5d2dcdda45660965a79f7ca01a8d779c71da39 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 15:25:01 -0700 Subject: [PATCH 06/20] Update docs/getting-started.rst --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 69f06715..274453b7 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -53,7 +53,7 @@ Installation Note that ``xesmf`` is an optional dependency, which is required for using ``xesmf`` based horizontal regridding APIs in ``xcdat``. ``xesmf`` is not currently supported - on `windows`_ because ``esmpy`` is not yet available on these platforms. Windows + on `windows`_ because ``esmpy`` is not yet available on this platform. Windows users can try `WSL2`_ as a workaround. .. _windows: https://github.com/conda-forge/esmf-feedstock/issues/64 From 4eae1034c323255caa9b7902240fc6ca9643cca5 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 15:33:51 -0700 Subject: [PATCH 07/20] Unpin gsw-xarray --- conda-env/dev.yml | 2 +- docs/getting-started.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/conda-env/dev.yml b/conda-env/dev.yml index 1d84b2db..868e08af 100644 --- a/conda-env/dev.yml +++ b/conda-env/dev.yml @@ -31,7 +31,7 @@ dependencies: - nbsphinx - pandoc - ipython >=8.0.0 # Required for nbsphinx syntax highlighting - - gsw-xarray=0.3.0 # Required for vertical regridding example + - gsw-xarray # Required for vertical regridding example # Quality Assurance # ================== - types-python-dateutil diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 274453b7..b720a2e1 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -13,7 +13,8 @@ Prerequisites pages if you aren't familiar with ``xarray``. 2. xCDAT is distributed available on Anaconda. We recommend installing `mamba`_, which - is a drop-in replacement of ``conda``. + is a drop-in replacement of ``conda``. Note, ``conda`` commands are still available + with Mambaforge. We recommend following these steps to install Mambaforge (Linux example): From 1ec2c6a1d7ad7f676b866ed20e9f50ba39a17f46 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 15:49:52 -0700 Subject: [PATCH 08/20] Keep doc dependencies pinned --- conda-env/dev.yml | 16 ++++++++-------- docs/getting-started.rst | 7 ++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/conda-env/dev.yml b/conda-env/dev.yml index 868e08af..fa86cc2c 100644 --- a/conda-env/dev.yml +++ b/conda-env/dev.yml @@ -24,14 +24,14 @@ dependencies: - nc-time-axis=1.4.1 # Documentation # ================== - - sphinx >=5.0.0 - - sphinx-autosummary-accessors - - sphinx-book-theme - - sphinx-copybutton - - nbsphinx - - pandoc - - ipython >=8.0.0 # Required for nbsphinx syntax highlighting - - gsw-xarray # Required for vertical regridding example + - sphinx=5.3.0 + - sphinx-autosummary-accessors=2022.4.0 + - sphinx-book-theme=1.0.1 + - sphinx-copybutton=0.5.1 + - nbsphinx=0.9.1 + - pandoc=3.1.1 + - ipython=8.11.0 # Required for nbsphinx syntax highlighting + - gsw-xarray=0.3.0 # Required for vertical regridding example # Quality Assurance # ================== - types-python-dateutil diff --git a/docs/getting-started.rst b/docs/getting-started.rst index b720a2e1..c954c288 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -12,9 +12,9 @@ Prerequisites - We highly recommend visiting the `xarray tutorial`_ and `xarray documentation`_ pages if you aren't familiar with ``xarray``. -2. xCDAT is distributed available on Anaconda. We recommend installing `mamba`_, which - is a drop-in replacement of ``conda``. Note, ``conda`` commands are still available - with Mambaforge. +2. xCDAT is distributed available on the Anaconda `conda-forge`_ channel. We recommend + installing `mamba`_ to access xCDAT. mamba is is a drop-in replacement of conda. + Note, ``conda`` commands are still available with Mambaforge. We recommend following these steps to install Mambaforge (Linux example): @@ -32,6 +32,7 @@ Prerequisites respond with ``yes`` so ``conda`` and ``mamba`` commands are available on initializing a new bash terminal. +.. _conda-forge: https://anaconda.org/conda-forge/xcdat .. _mamba: https://mamba.readthedocs.io/en/latest/index.html .. _xarray tutorial: https://tutorial.xarray.dev/intro.html .. _xarray documentation: https://docs.xarray.dev/en/stable/getting-started-guide/index.html From 3eeaf3479e94b51aed7baa7ac9b7d452ed86aac5 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:03:49 -0700 Subject: [PATCH 09/20] Apply suggestions from code review --- CONTRIBUTING.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 3c8fbe1b..66025892 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -139,7 +139,6 @@ Local Development $ wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh $ bash ./Mambaforge-Linux-x86_64.sh - Do you wish the installer to initialize Miniconda3 by running conda init? [yes|no] yes MacOS @@ -147,7 +146,6 @@ Local Development $ wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-MacOSX-x86_64.sh $ bash ./Mambaforge-MacOSX-x86_64.sh - Do you wish the installer to initialize Miniconda3 by running conda init? [yes|no] yes 2. Fork the ``xcdat`` repo on GitHub. @@ -160,7 +158,7 @@ Local Development 4. Open ``.vscode/xcdat.code-settings`` in VS Code -5. Create and activate mamba development environment:: +5. Create and activate the development environment:: $ cd xcdat $ mamba env create -f conda-env/dev.yml From a3c114f21ab0a3d0c87ffae873a5bffdf1c59104 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:04:27 -0700 Subject: [PATCH 10/20] Update docs/getting-started.rst --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index c954c288..ebca8b48 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -12,7 +12,7 @@ Prerequisites - We highly recommend visiting the `xarray tutorial`_ and `xarray documentation`_ pages if you aren't familiar with ``xarray``. -2. xCDAT is distributed available on the Anaconda `conda-forge`_ channel. We recommend +2. xCDAT is distributed on the Anaconda `conda-forge`_ channel. We recommend installing `mamba`_ to access xCDAT. mamba is is a drop-in replacement of conda. Note, ``conda`` commands are still available with Mambaforge. From b6d3eadf44b10a8b23d99c79efb802d39422652c Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:14:20 -0700 Subject: [PATCH 11/20] Update docs/getting-started.rst --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index ebca8b48..b3b77576 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -13,7 +13,7 @@ Prerequisites pages if you aren't familiar with ``xarray``. 2. xCDAT is distributed on the Anaconda `conda-forge`_ channel. We recommend - installing `mamba`_ to access xCDAT. mamba is is a drop-in replacement of conda. + installing `mamba`_ to access xCDAT. mamba is a drop-in replacement of conda. Note, ``conda`` commands are still available with Mambaforge. We recommend following these steps to install Mambaforge (Linux example): From f02cfa971f11a1f4254c970f2196cccef84d396a Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:15:01 -0700 Subject: [PATCH 12/20] Update docs/getting-started.rst --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index b3b77576..ad165ffc 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -13,7 +13,7 @@ Prerequisites pages if you aren't familiar with ``xarray``. 2. xCDAT is distributed on the Anaconda `conda-forge`_ channel. We recommend - installing `mamba`_ to access xCDAT. mamba is a drop-in replacement of conda. + installing `mamba`_ to access xCDAT. Mamba is a drop-in replacement of conda. Note, ``conda`` commands are still available with Mambaforge. We recommend following these steps to install Mambaforge (Linux example): From 5be8665ccfb9cb2a49aad102442cbf12799a1e62 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:19:31 -0700 Subject: [PATCH 13/20] Revert doc changes for mamba --- CONTRIBUTING.rst | 16 ++++++----- docs/getting-started.rst | 59 ++++++++++++++++++++-------------------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 66025892..1d79514d 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -137,15 +137,17 @@ Local Development Linux :: - $ wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh - $ bash ./Mambaforge-Linux-x86_64.sh + $ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + $ bash ./Miniconda3-latest-Linux-x86_64.sh + Do you wish the installer to initialize Miniconda3 by running conda init? [yes|no] yes MacOS :: - $ wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-MacOSX-x86_64.sh - $ bash ./Mambaforge-MacOSX-x86_64.sh + $ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + $ bash ./Miniconda3-latest-MacOSX-x86_64.sh + Do you wish the installer to initialize Miniconda3 by running conda init? [yes|no] yes 2. Fork the ``xcdat`` repo on GitHub. @@ -158,11 +160,11 @@ Local Development 4. Open ``.vscode/xcdat.code-settings`` in VS Code -5. Create and activate the development environment:: +5. Create and activate Conda development environment:: $ cd xcdat - $ mamba env create -f conda-env/dev.yml - $ mamba activate xcdat_dev + $ conda env create -f conda-env/dev.yml + $ conda activate xcdat_dev 6. Set VS Code Python interpretor to ``xcdat_dev`` diff --git a/docs/getting-started.rst b/docs/getting-started.rst index ad165ffc..0a8ffa71 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -12,64 +12,63 @@ Prerequisites - We highly recommend visiting the `xarray tutorial`_ and `xarray documentation`_ pages if you aren't familiar with ``xarray``. -2. xCDAT is distributed on the Anaconda `conda-forge`_ channel. We recommend - installing `mamba`_ to access xCDAT. Mamba is a drop-in replacement of conda. - Note, ``conda`` commands are still available with Mambaforge. +2. xCDAT is distributed through conda, which is available through Anaconda and Miniconda. - We recommend following these steps to install Mambaforge (Linux example): + We recommend following these steps to install Miniconda: .. code-block:: console - wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + bash Miniconda3-latest-Linux-x86_64.sh - bash ./Mambaforge-Linux-x86_64.sh + Then follow the instructions for installation. To have conda added to + your path you will need to type ``yes`` in response to ``"Do you wish the + installer to initialize Miniconda3 by running conda init?"`` (we recommend + that you do this). Note that this will modify your shell profile (e.g., + ``~/.bashrc``) to add ``conda`` to your path. - When you see: :: + Note: After installation completes you may need to type ``bash`` to + restart your shell (if you use bash). Alternatively, you can log out and + log back in. - by running conda init? [yes|no] - [no] >>> yes - - respond with ``yes`` so ``conda`` and ``mamba`` commands are available on - initializing a new bash terminal. - -.. _conda-forge: https://anaconda.org/conda-forge/xcdat -.. _mamba: https://mamba.readthedocs.io/en/latest/index.html .. _xarray tutorial: https://tutorial.xarray.dev/intro.html .. _xarray documentation: https://docs.xarray.dev/en/stable/getting-started-guide/index.html Installation ------------ -1. Create a mamba environment from scratch with ``xcdat`` (`mamba create`_) +1. Create a conda environment from scratch with ``xcdat`` (`conda create`_) - We recommend using the mamba environment creation procedure to install ``xcdat``. - The advantage with following this approach is that mamba will attempt to resolve - dependencies (e.g. ``python >= 3.9``) for compatibility. + We recommend using the Conda environment creation procedure to install ``xcdat``. + The advantage with following this approach is that Conda will attempt to resolve + dependencies (e.g. ``python >= 3.8``) for compatibility. - To create an ``xcdat`` environment with ``xesmf`` (a recommended dependency), run: + To create an ``xcdat`` conda environment with ``xesmf`` (a recommended dependency), + run: .. code-block:: console - >>> mamba create -n -c conda-forge xcdat xesmf - >>> mamba activate + >>> conda create -n -c conda-forge xcdat xesmf + >>> conda activate Note that ``xesmf`` is an optional dependency, which is required for using ``xesmf`` based horizontal regridding APIs in ``xcdat``. ``xesmf`` is not currently supported - on `windows`_ because ``esmpy`` is not yet available on this platform. Windows - users can try `WSL2`_ as a workaround. + on `osx-arm64`_ or `windows`_ because ``esmpy`` is not yet available on these + platforms. Windows users can try `WSL2`_ as a workaround. .. _windows: https://github.com/conda-forge/esmf-feedstock/issues/64 +.. _osx-arm64: https://github.com/conda-forge/esmf-feedstock/issues/74 .. _WSL2: https://docs.microsoft.com/en-us/windows/wsl/install -2. Install ``xcdat`` in an existing mamba environment (`mamba install`_) +2. Install ``xcdat`` in an existing conda environment (`conda install`_) - You can also install ``xcdat`` in an existing mamba environment, granted that mamba + You can also install ``xcdat`` in an existing Conda environment, granted that Conda is able to resolve the compatible dependencies. .. code-block:: console - >>> mamba activate - >>> mamba install -c conda-forge xcdat xesmf + >>> conda activate + >>> conda install -c conda-forge xcdat xesmf Note: As above, ``xesmf`` is an optional dependency. @@ -81,5 +80,5 @@ Installation - ``matplotlib``: a library for creating visualizations in Python. - ``cartopy``: an add-on package for ``matplotlib`` and specialized for geospatial data processing. -.. _mamba create: https://fig.io/manual/mamba/create -.. _mamba install: https://fig.io/manual/mamba/install +.. _conda create: https://docs.conda.io/projects/conda/en/latest/commands/create.html?highlight=create +.. _conda install: https://docs.conda.io/projects/conda/en/latest/commands/install.html?highlight=install From 54ee4edd84cc86143bf84ce80ad2fc3161544996 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:23:50 -0700 Subject: [PATCH 14/20] Replace Python 3.8 with 3.11 in build workflow --- .github/workflows/build_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index a62120b0..dc0f51cf 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -50,7 +50,7 @@ jobs: shell: bash -l {0} strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11"] steps: - id: skip_check uses: fkirc/skip-duplicate-actions@master From 8fffc6232135851194f4eba5fa3f83ccefc88b4d Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:32:49 -0700 Subject: [PATCH 15/20] Ignore dependency updates with gh actions --- .github/workflows/build_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index dc0f51cf..53408f81 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -98,7 +98,7 @@ jobs: - if: $${{ steps.skip_check.outputs.should_skip != 'true' || steps.cache.outputs.cache-hit != 'true' }} name: Update environment - run: mamba env update -n xcdat_ci -f conda-env/ci.yml + run: mamba env update -n xcdat_ci -f conda-env/ci.yml --no-update-deps - if: ${{ steps.skip_check.outputs.should_skip != 'true' }} name: Install xcdat From 54231dd497464f3b477809b34d18c744dbb3860d Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:38:13 -0700 Subject: [PATCH 16/20] Fix line for updating mamba env --- .github/workflows/build_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index 53408f81..e68e2c51 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -98,7 +98,7 @@ jobs: - if: $${{ steps.skip_check.outputs.should_skip != 'true' || steps.cache.outputs.cache-hit != 'true' }} name: Update environment - run: mamba env update -n xcdat_ci -f conda-env/ci.yml --no-update-deps + run: mamba update -n xcdat_ci -f conda-env/ci.yml --freeze-installed - if: ${{ steps.skip_check.outputs.should_skip != 'true' }} name: Install xcdat From 21eb4ad83d077b56add66fa53b1ffc4ac3cceb27 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:41:01 -0700 Subject: [PATCH 17/20] Reset cache --- .github/workflows/build_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index e68e2c51..7b34a19e 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -94,7 +94,7 @@ jobs: steps.get-date.outputs.today }}-${{hashFiles('conda-env/ci.yml') }}-${{ env.CACHE_NUMBER}} env: # Increase this value to reset cache if conda-env/ci.yml has not changed in the workflow - CACHE_NUMBER: 0 + CACHE_NUMBER: 1 - if: $${{ steps.skip_check.outputs.should_skip != 'true' || steps.cache.outputs.cache-hit != 'true' }} name: Update environment From 07c24252a835231d16a5ae50dab076698ab7f12b Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:43:40 -0700 Subject: [PATCH 18/20] Revert mamba changes --- .github/workflows/build_workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index 7b34a19e..dc0f51cf 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -94,11 +94,11 @@ jobs: steps.get-date.outputs.today }}-${{hashFiles('conda-env/ci.yml') }}-${{ env.CACHE_NUMBER}} env: # Increase this value to reset cache if conda-env/ci.yml has not changed in the workflow - CACHE_NUMBER: 1 + CACHE_NUMBER: 0 - if: $${{ steps.skip_check.outputs.should_skip != 'true' || steps.cache.outputs.cache-hit != 'true' }} name: Update environment - run: mamba update -n xcdat_ci -f conda-env/ci.yml --freeze-installed + run: mamba env update -n xcdat_ci -f conda-env/ci.yml - if: ${{ steps.skip_check.outputs.should_skip != 'true' }} name: Install xcdat From 7401005fb50d61d63802a532466500461aeaf5a7 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:44:49 -0700 Subject: [PATCH 19/20] Make sure Python version does not change --- .github/workflows/build_workflow.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index dc0f51cf..a523b5f3 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -98,7 +98,9 @@ jobs: - if: $${{ steps.skip_check.outputs.should_skip != 'true' || steps.cache.outputs.cache-hit != 'true' }} name: Update environment - run: mamba env update -n xcdat_ci -f conda-env/ci.yml + run: | + mamba env update -n xcdat_ci -f conda-env/ci.yml + mamba install -c conda-forge python=${{ matrix.python-version }} - if: ${{ steps.skip_check.outputs.should_skip != 'true' }} name: Install xcdat From 96d4fe7f1f51ae1be415993c662acc8a16feb1c2 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 22 Jun 2023 16:45:35 -0700 Subject: [PATCH 20/20] Add comment --- .github/workflows/build_workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index a523b5f3..73391e1c 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -100,6 +100,7 @@ jobs: name: Update environment run: | mamba env update -n xcdat_ci -f conda-env/ci.yml + # Make sure the Python version in the env matches the current matrix version. mamba install -c conda-forge python=${{ matrix.python-version }} - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}