Merge branch 'main-upstream' into stable

OSOceanAcoustics · Aug 30, 2023 · 27bd8f2 · 27bd8f2
2 parents 7baa474 + 254ee10
commit 27bd8f2
Show file tree

Hide file tree

Showing 89 changed files with 21,108 additions and 8,355 deletions.
diff --git a/.ci_helpers/py3.10.yaml b/.ci_helpers/py3.10.yaml
@@ -3,14 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.10
-  - dask
-  - netCDF4
-  - pynmea2
-  - pytz
-  - scipy
-  - xarray==2022.3.0
-  - zarr
-  - fsspec
-  - s3fs==2022.5.0
-  - matplotlib-base
-  - cmocean
+  - pip
+  - pip:
+    - -r ../requirements.txt
diff --git a/.ci_helpers/py3.11.yaml b/.ci_helpers/py3.11.yaml
@@ -0,0 +1,8 @@
+name: echopype
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11
+  - pip
+  - pip:
+    - -r ../requirements.txt
diff --git a/.ci_helpers/py3.8.yaml b/.ci_helpers/py3.8.yaml
@@ -3,14 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.8
-  - dask
-  - netCDF4
-  - pynmea2
-  - pytz
-  - scipy
-  - xarray==2022.3.0
-  - zarr
-  - fsspec
-  - s3fs==2022.5.0
-  - matplotlib-base
-  - cmocean
+  - pip
+  - pip:
+    - -r ../requirements.txt
diff --git a/.ci_helpers/py3.9.yaml b/.ci_helpers/py3.9.yaml
@@ -3,14 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.9
-  - dask
-  - netCDF4
-  - pynmea2
-  - pytz
-  - scipy
-  - xarray==2022.3.0
-  - zarr
-  - fsspec
-  - s3fs==2022.5.0
-  - matplotlib-base
-  - cmocean
+  - pip
+  - pip:
+    - -r ../requirements.txt
diff --git a/.ci_helpers/run-test.py b/.ci_helpers/run-test.py
@@ -33,7 +33,7 @@
     "echodata": {},
     "mask": {},
     "metrics": {},
-    "preprocess": {},
+    "qc": {},
     "utils": {},
     "visualize": {},
 }

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -16,18 +16,19 @@ jobs:
   test:
     name: ${{ matrix.python-version }}-build
     runs-on: ubuntu-20.04
-    if: "!contains(github.event.head_commit.message, '[skip ci]')"
+    if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-        - python-version: 3.8
-          experimental: false
-        - python-version: 3.9
-          experimental: false
-        - python-version: "3.10"
-          experimental: true
+        python-version: ["3.8", "3.9", "3.10"]
+        runs-on: [ubuntu-latest]
+        experimental: [false]
+        # TODO: Uncomment when netcdf is unpinned
+        # include:
+        # - runs-on: ubuntu-latest
+        #   python-version: "3.11"
+        #   experimental: true
     services:
       # TODO: figure out how to update tag when there's a new one
       minio:
@@ -47,19 +48,19 @@ jobs:
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
       - name: Setup micromamba
-        uses: mamba-org/provision-with-micromamba@v15
+        uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: .ci_helpers/py${{ matrix.python-version }}.yaml
           environment-name: ${{ env.CONDA_ENV }}
-          cache-env: true
-          cache-env-key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles(format('.ci_helpers/py{0}.yaml', matrix.python-version)) }}
-      - name: Print conda env
+          cache-environment: true
+          post-cleanup: 'all'
+      - name: Print conda environment
         shell: bash -l {0}
         run: |
-          conda info
-          conda list
+          micromamba info
+          micromamba list
       - name: Remove docker-compose python
-        if: ${{ matrix.python-version == '3.10' }}
+        if: ${{ matrix.python-version == '3.10' || matrix.python-version == '3.11' }}
         shell: bash -l {0}
         run: sed -i "/docker-compose/d" requirements-dev.txt
       - name: Install dev tools
@@ -86,7 +87,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: ci_test_log
-          path: ci_test_log.log
+          path: ci_${{ matrix.python-version }}_test_log.log
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

diff --git a/.github/workflows/packit.yaml b/.github/workflows/packit.yaml
@@ -20,7 +20,7 @@ jobs:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v4.5.0
+      uses: actions/setup-python@v4.7.0
       with:
         python-version: 3.9
 
@@ -52,7 +52,7 @@ jobs:
     needs: build-artifact
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/setup-python@v4.5.0
+    - uses: actions/setup-python@v4.7.0
       name: Install Python
       with:
         python-version: 3.9

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -10,20 +10,24 @@ env:
 
 jobs:
   test:
-    name: ${{ matrix.python-version }}-build
-    runs-on: ubuntu-20.04
-    if: "!contains(github.event.pull_request.title, '[skip ci]')"
+    name: ${{ matrix.python-version }}--${{ matrix.runs-on }}
+    runs-on: ${{ matrix.runs-on }}
+    if: ${{ !contains(github.event.pull_request.title, '[skip ci]') }}
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-        - python-version: 3.8
-          experimental: false
-        - python-version: 3.9
-          experimental: false
-        - python-version: "3.10"
-          experimental: true
+        python-version: ["3.8", "3.9", "3.10"]
+        runs-on: [ubuntu-latest]
+        experimental: [false]
+        # TODO: Uncomment when netcdf is unpinned
+        # include:
+        # - runs-on: ubuntu-latest
+        #   python-version: "3.11"
+        #   experimental: true
+    defaults:
+      run:
+        shell: bash -l {0}
     services:
       # TODO: figure out how to update tag when there's a new one
       minio:
@@ -43,60 +47,58 @@ jobs:
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
       - name: Setup micromamba
-        uses: mamba-org/provision-with-micromamba@v15
+        uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: .ci_helpers/py${{ matrix.python-version }}.yaml
           environment-name: ${{ env.CONDA_ENV }}
-          cache-env: true
-          cache-env-key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles(format('.ci_helpers/py{0}.yaml', matrix.python-version)) }}
-      - name: Print conda env
-        shell: bash -l {0}
+          cache-environment: true
+          post-cleanup: 'all'
+      - name: Print conda environment
         run: |
-          conda info
-          conda list
+          micromamba info
+          micromamba list
       - name: Remove docker-compose python
-        if: ${{ matrix.python-version == '3.10' }}
-        shell: bash -l {0}
+        if: ${{ matrix.python-version == '3.10' || matrix.python-version == '3.11' }}
         run: sed -i "/docker-compose/d" requirements-dev.txt
       - name: Install dev tools
-        shell: bash -l {0}
         run: |
           micromamba install -c conda-forge -n ${{ env.CONDA_ENV }} --yes --file requirements-dev.txt
+      # We only want to install this on one run, because otherwise we'll have
+      # duplicate annotations.
+      - name: Install error reporter
+        if: ${{ matrix.python-version == '3.9' }}
+        run: |
+          python -m pip install pytest-github-actions-annotate-failures
       - name: Install echopype
-        shell: bash -l {0}
         run: |
           python -m pip install -e .[plot]
       - name: Copying test data to services
-        shell: bash -l {0}
         run: |
           python .ci_helpers/docker/setup-services.py --deploy --data-only --http-server ${{ job.services.httpserver.id }}
 
           # Check data endpoint
           curl http://localhost:8080/data/
       - name: Finding changed files
         id: files
-        uses: lsetiawan/get-changed-files@pr_target
+        uses: Ana06/get-changed-files@v2.2.0
         with:
           format: 'csv'
       - name: Print Changed files
-        shell: bash -l {0}
         run: echo "${{ steps.files.outputs.added_modified_renamed }}"
       - name: Running all Tests
         if: contains(github.event.pull_request.title, '[all tests ci]')
-        shell: bash -l {0}
         run: |
           pytest -vvv -rx --numprocesses=${{ env.NUM_WORKERS }} --max-worker-restart=3 --cov=echopype --cov-report=xml --log-cli-level=WARNING --disable-warnings |& tee ci_${{ matrix.python-version }}_test_log.log
       - name: Running Tests
-        if: "!contains(github.event.pull_request.title, '[all tests ci]')"
-        shell: bash -l {0}
+        if: ${{ !contains(github.event.pull_request.title, '[all tests ci]') }}
         run: |
-          python .ci_helpers/run-test.py --pytest-args="--log-cli-level=WARNING,-vvv,-rx,--numprocesses=${{ env.NUM_WORKERS }},--max-worker-restart=3,--disable-warnings" --include-cov ${{ steps.files.outputs.added_modified_renamed }} |& tee ci_test_log.log
+          python .ci_helpers/run-test.py --pytest-args="--log-cli-level=WARNING,-vvv,-rx,--numprocesses=${{ env.NUM_WORKERS }},--max-worker-restart=3,--disable-warnings" --include-cov ${{ steps.files.outputs.added_modified_renamed }} |& tee ci_${{ matrix.python-version }}_test_log.log
       - name: Upload ci test log
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v3
         with:
           name: ci_test_log
-          path: ci_test_log.log
+          path: ci_${{ matrix.python-version }}_test_log.log
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml
@@ -24,7 +24,7 @@ jobs:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v4.5.0
+      uses: actions/setup-python@v4.7.0
       with:
         python-version: 3.9
 
@@ -56,7 +56,7 @@ jobs:
     needs: build-artifact
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/setup-python@v4.5.0
+    - uses: actions/setup-python@v4.7.0
       name: Install Python
       with:
         python-version: 3.9
@@ -69,7 +69,7 @@ jobs:
         ls -ltrh
         ls -ltrh dist
     - name: Publish to Test PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.1
+      uses: pypa/gh-action-pypi-publish@v1.8.10
       with:
         password: ${{ secrets.TEST_PYPI_API_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
@@ -104,6 +104,6 @@ jobs:
         name: releases
         path: dist
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.1
+      uses: pypa/gh-action-pypi-publish@v1.8.10
       with:
         password: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml
@@ -46,7 +46,7 @@ jobs:
             # Check data endpoint
             curl http://localhost:8080/data/
         - name: Setup Python
-          uses: actions/setup-python@v4.5.0
+          uses: actions/setup-python@v4.7.0
           with:
             python-version: ${{ matrix.python-version }}
             architecture: x64

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -24,17 +24,17 @@ repos:
         args: ["--profile", "black", "--filter-files"]
 
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.7.0
     hooks:
       - id: black
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.4
+    rev: v2.2.5
     hooks:
       - id: codespell
         # Checks spelling in `docs/source` and `echopype` dirs ONLY

diff --git a/docs/source/_config.yml b/docs/source/_config.yml
@@ -18,8 +18,8 @@ execute:
 #     targetname: book.tex
 
 # Add a bibtex file so that we can create citations
-# bibtex_bibfiles:
-#   - references.bib
+bibtex_bibfiles:
+  - references.bib
 
 # Information about where the book exists on the web
 repository:
@@ -39,13 +39,15 @@ sphinx:
     'sphinx_automodapi.automodapi',
     'numpydoc',
     # 'sphinx.ext.autodoc',
+    'sphinxcontrib.bibtex',
     'sphinx.ext.intersphinx',
     'sphinx.ext.mathjax',
     'sphinx.ext.ifconfig',
     'sphinx.ext.githubpages',
     'sphinxcontrib.mermaid'
   ]
-  # config:
+  config:
+    bibtex_reference_style: label
   #   # https://github.com/executablebooks/jupyter-book/issues/1186
   #   # https://sphinx-book-theme.readthedocs.io/en/latest/customize/sidebar-secondary.html
   #   html_theme_options:

diff --git a/docs/source/_toc.yml b/docs/source/_toc.yml
@@ -12,20 +12,25 @@ parts:
   chapters:
   - file: convert
   - file: open-converted
-  - file: process
   - file: data-format
-    title: Data formats
+    title: Raw data formats
     sections:
+    - file: data-format-sonarnetcdf4
     - file: data-format-raw
-    - file: data-format-processed
-    - file: data-format-5to6
+    - file: data-format-changes
+  - file: data-proc
+    title: Data processing
+    sections:
+    - file: data-proc-func
+    - file: data-proc-additional
   - file: viz
   - file: processing-levels
     title: Processing levels
-- caption: Help & reference
+- caption: Help & references
   chapters:
-  - file: api
   - file: whats-new
-  - file: roadmap
   - file: contributing
+  - file: api
+  - file: zbibliography
+  - file: roadmap
   - file: resources