Merge pull request #785 from OSOceanAcoustics/dev

Release/v0.6.2

.ci_helpers/py3.10.yaml

@@ -8,7 +8,7 @@ dependencies:
   - pynmea2
   - pytz
   - scipy
-  - xarray
+  - xarray==2022.3.0
   - zarr
   - fsspec
   - s3fs==2022.5.0

.ci_helpers/py3.8.yaml

@@ -8,7 +8,7 @@ dependencies:
   - pynmea2
   - pytz
   - scipy
-  - xarray
+  - xarray==2022.3.0
   - zarr
   - fsspec
   - s3fs==2022.5.0

.ci_helpers/py3.9.yaml

@@ -8,7 +8,7 @@ dependencies:
   - pynmea2
   - pytz
   - scipy
-  - xarray
+  - xarray==2022.3.0
   - zarr
   - fsspec
   - s3fs==2022.5.0

docs/source/contributing.rst

@@ -91,7 +91,14 @@ Create a `conda <https://docs.conda.io>`_ environment for echopype development
     # install echopype in editable mode (setuptools "develop mode")
     # plot is an extra set of requirements that can be used for plotting.
     # the command will install all the dependencies along with plotting dependencies.
-    pip install -e .[plot]
+    pip install -e ".[plot]"
+
+.. note::
+
+    Try using `mamba <https://mamba.readthedocs.io>`_ instead of ``conda``
+    if the ``conda create`` and ``conda install`` step fail or take too long.
+    ``Mamba`` is a drop-in replacement for conda environment creation and package
+    installation that is typically faster than conda.
 
 See the :doc:`installation` page to simply install the latest echopype release from conda or PyPI.
 
@@ -102,13 +109,6 @@ Tests and test infrastructure
 Test data files
 ~~~~~~~~~~~~~~~
 
-.. attention::
-
-    Echopype previously used Git LFS for managing and accessing large test data files.
-    We have deprecated its use starting with echopype version 0.5.0. The files
-    in https://github.com/OSOceanAcoustics/echopype/tree/main/echopype/test_data
-    are also being deprecated.
-
 Test echosounder data files are managed in a private Google Drive folder and
 made available via the `cormorack/http <https://hub.docker.com/r/cormorack/http>`_
 Docker image on Docker hub; the image is rebuilt daily when new test data are added
@@ -181,14 +181,12 @@ The entire test suite can be a bit slow, taking up to 40 minutes or more.
 To mitigate this, the CI default is to run tests only for subpackages that
 were modified in the PR; this is done via ``.ci_helpers/run-test.py``
 (see the `Running the tests`_ section). To have the CI execute the
-entire test suite, add the GitHub label ``Needs Complete Testing`` to the
-PR before submitting it.
-
+entire test suite, add the string "[all tests ci]" to the PR title.
 Under special circumstances, when the submitted changes have a
 very limited scope (such as contributions to the documentation)
 or you know exactly what you're doing
 (you're a seasoned echopype contributor), the CI can be skipped.
-This is done by including the string "[skip ci]" in your last commit's message.
+This is done by adding the string "[skip ci]" to the PR title.
 
 
 Documentation development

docs/source/installation.rst

@@ -5,7 +5,7 @@ Installation and Examples
 Installation
 ------------
 
-Echopype is available and tested for Python>=3.7. The latest release
+Echopype is available and tested for Python>=3.8. The latest release
 can be installed from `PyPI <https://pypi.org/project/echopype/>`_:
 
 .. code-block:: console

docs/source/whats-new.md

@@ -4,7 +4,42 @@ What's new
 See [GitHub releases page](https://github.com/OSOceanAcoustics/echopype/releases) for the complete history.
 
 
-# v0.6.1 (2022 July 2)
+# v0.6.2 (2022 August 13)
+
+## Overview
+
+This is a minor release that includes a few new features and memory efficiency-related changes that make echopype better.
+
+## New features
+- Add a new subpackage `consolidate` that contains functions to consolidate data across the calibrated Sv dataset and the corresponding raw-converted file
+- Add function `consolidate.add_location` to interpolate location to calibrated dataset (#749)
+- Add function `consolidate.add_depth` to convert `range_meter` to `depth` with information on transducer tilt and depth (#738)
+- Move function that swaps the channel with frequency dimension (now as `consolidate.swap_dims_channel_frequency`) (#738)
+- Add new functionality to allow control of logging outputs (#772)
+
+## Under the hood enhancements
+- Improve memory usage while converting files that require significant NaN-padding and previously would incur very large memory expansion (#774)
+  - This is achieved by directly writing variables that may incur a large memory expansion into a temporary zarr store
+  - Beta function that will benefit from user feedback
+- Overhaul access pattern for EchoData (#762)
+  - Remove previous access pattern for different groups in the raw-converted file
+  - Starting from this release all groups are accessed with `echodata["GROUP_PATH"]`, e.g., `echodata["Platform"]`, `echodata["Sonar/Beam_group1"]`, etc.
+- Make long_name in ds_power for EK80 consistent with other sonar model (#771)
+- Modify `set_beam()` so it returns a list (#780)
+- Change the order in `_save_groups_to_file` so the order of groups is preserved when opening a converted netcdf file (#779)
+- Remove the user option to select NMEA sentences in `open_raw` to ensure raw data is preserved (#778)
+
+## Infrastructure
+- Update CI set up to avoid exceeding GitHub actions memory limitation (#761)
+  - decrease the number of workers from 4 to 2
+
+
+
+
+
+
+
+# v0.6.1 (2022 July 7)
 
 ## Overview
 

echopype/__init__.py

@@ -2,9 +2,21 @@
 
 from _echopype_version import version as __version__  # noqa
 
-from . import calibrate, preprocess, utils
+from . import calibrate, consolidate, preprocess, utils
 from .convert.api import open_raw
 from .echodata.api import open_converted
 from .echodata.combine import combine_echodata
+from .utils.log import verbose
 
-__all__ = ["open_raw", "open_converted", "combine_echodata", "calibrate", "preprocess", "utils"]
+verbose(override=True)
+
+__all__ = [
+    "open_raw",
+    "open_converted",
+    "combine_echodata",
+    "calibrate",
+    "consolidate",
+    "preprocess",
+    "utils",
+    "verbose",
+]

echopype/calibrate/api.py

@@ -1,8 +1,7 @@
-import warnings
-
 import xarray as xr
 
 from ..echodata import EchoData
+from ..utils.log import _init_logger
 from ..utils.prov import echopype_prov_attrs, source_files_vars
 from .calibrate_azfp import CalibrateAZFP
 from .calibrate_ek import CalibrateEK60, CalibrateEK80
@@ -16,6 +15,8 @@
     "EA640": CalibrateEK80,
 }
 
+logger = _init_logger(__name__)
+
 
 def _compute_cal(
     cal_type,
@@ -39,12 +40,12 @@ def _compute_cal(
             )
     elif echodata.sonar_model in ("EK60", "AZFP"):
         if waveform_mode is not None and waveform_mode != "CW":
-            warnings.warn(
+            logger.warning(
                 "This sonar model transmits only narrowband signals (waveform_mode='CW'). "
                 "Calibration will be in CW mode",
             )
         if encode_mode is not None and encode_mode != "power":
-            warnings.warn(
+            logger.warning(
                 "This sonar model only record data as power or power/angle samples "
                 "(encode_mode='power'). Calibration will be done on the power samples.",
             )
@@ -104,9 +105,9 @@ def add_attrs(cal_type, ds):
     prov_dict["processing_function"] = f"calibrate.compute_{cal_type}"
     cal_ds = cal_ds.assign_attrs(prov_dict)
 
-    if "water_level" in echodata.platform.data_vars.keys():
+    if "water_level" in echodata["Platform"].data_vars.keys():
         # add water_level to the created xr.Dataset
-        cal_ds["water_level"] = echodata.platform.water_level
+        cal_ds["water_level"] = echodata["Platform"].water_level
 
     return cal_ds
 

echopype/calibrate/calibrate_azfp.py

@@ -34,13 +34,15 @@ def get_cal_params(self, cal_params):
         self.cal_params["equivalent_beam_angle"] = (
             cal_params["equivalent_beam_angle"]
             if "equivalent_beam_angle" in cal_params
-            else self.echodata.beam["equivalent_beam_angle"]
+            else self.echodata["Sonar/Beam_group1"]["equivalent_beam_angle"]
         )
 
         # Get params from the Vendor_specific group
         for p in ["EL", "DS", "TVR", "VTX", "Sv_offset"]:
             # substitute if None in user input
-            self.cal_params[p] = cal_params[p] if p in cal_params else self.echodata.vendor[p]
+            self.cal_params[p] = (
+                cal_params[p] if p in cal_params else self.echodata["Vendor_specific"][p]
+            )
 
     def get_env_params(self):
         """Get env params using user inputs or values from data file.
@@ -53,7 +55,7 @@ def get_env_params(self):
         self.env_params["temperature"] = (
             self.env_params["temperature"]
             if "temperature" in self.env_params
-            else self.echodata.environment["temperature"]
+            else self.echodata["Environment"]["temperature"]
         )
 
         # Salinity and pressure always come from user input
@@ -71,7 +73,7 @@ def get_env_params(self):
             formula_source="AZFP",
         )
         self.env_params["sound_absorption"] = uwa.calc_absorption(
-            frequency=self.echodata.beam["frequency_nominal"],
+            frequency=self.echodata["Sonar/Beam_group1"]["frequency_nominal"],
             temperature=self.env_params["temperature"],
             salinity=self.env_params["salinity"],
             pressure=self.env_params["pressure"],
@@ -108,10 +110,10 @@ def _cal_power(self, cal_type, **kwargs):
         # Compute derived params
 
         # Harmonize time coordinate between Beam_groupX data and env_params
-        # Use self.echodata.beam because complex sample is always in Beam_group1
+        # Use self.echodata["Sonar/Beam_group1"] because complex sample is always in Beam_group1
         for p in self.env_params.keys():
             self.env_params[p] = self.echodata._harmonize_env_param_time(
-                self.env_params[p], ping_time=self.echodata.beam.ping_time
+                self.env_params[p], ping_time=self.echodata["Sonar/Beam_group1"].ping_time
             )
 
         # TODO: take care of dividing by zero encountered in log10
@@ -122,7 +124,9 @@ def _cal_power(self, cal_type, **kwargs):
         # scaling factor (slope) in Fig.G-1, units Volts/dB], see p.84
         a = self.cal_params["DS"]
         EL = (
-            self.cal_params["EL"] - 2.5 / a + self.echodata.beam.backscatter_r / (26214 * a)
+            self.cal_params["EL"]
+            - 2.5 / a
+            + self.echodata["Sonar/Beam_group1"].backscatter_r / (26214 * a)
         )  # eq.(5)  # has beam dim due to backscatter_r
 
         if cal_type == "Sv":
@@ -136,7 +140,7 @@ def _cal_power(self, cal_type, **kwargs):
                 * np.log10(
                     0.5
                     * self.env_params["sound_speed"]
-                    * self.echodata.beam["transmit_duration_nominal"]
+                    * self.echodata["Sonar/Beam_group1"]["transmit_duration_nominal"]
                     * self.cal_params["equivalent_beam_angle"]
                 )
                 + self.cal_params["Sv_offset"]
@@ -155,7 +159,7 @@ def _cal_power(self, cal_type, **kwargs):
         out = out.merge(self.range_meter)
 
         # Add frequency_nominal to data set
-        out["frequency_nominal"] = self.echodata.beam["frequency_nominal"]
+        out["frequency_nominal"] = self.echodata["Sonar/Beam_group1"]["frequency_nominal"]
 
         # Add env and cal parameters
         out = self._add_params_to_output(out)