Changed parser in parse_azfp.py to maintain consistency over other …

…parsers. (#1135)

* replaced usage of xml.dom.minidom with xml.etree.ElementTree

* modified azfp parser and parsed all parameters

* refactored the parse_azfp code and added new tests

* small tweak in test, rename conversion function to _camel_to_snake

* updated parameter names

* Replace camel-to-snake conversion funcitonality (#1)

* Create utils camel-to-snake-case function in new misc.py, and use it in ek_raw_parsers

* Replace AZFP camel-to-snake function with new utils/misc.py function

* fixed minor error in code

* minor change in test_convert_azfp.py

* fixed failing tests

* fixed failing tests related to attribute name

* fixed minor bug

---------

Co-authored-by: Wu-Jung Lee <leewujung@gmail.com>
Co-authored-by: Emilio Mayorga <emiliomayorga@gmail.com>

echopype/calibrate/api.py

@@ -153,7 +153,7 @@ def compute_Sv(echodata: EchoData, **kwargs) -> xr.Dataset:
         - for EK60 echosounder, allowed parameters include:
           `"sa_correction"`, `"gain_correction"`, `"equivalent_beam_angle"`
         - for AZFP echosounder, allowed parameters include:
-          `"EL"`, `"DS"`, `"TVR"`, `"VTX"`, `"equivalent_beam_angle"`, `"Sv_offset"`
+          `"EL"`, `"DS"`, `"TVR"`, `"VTX0"`, `"equivalent_beam_angle"`, `"Sv_offset"`
 
         Passing in calibration parameters for other echosounders
         are not currently supported.
@@ -242,7 +242,7 @@ def compute_TS(echodata: EchoData, **kwargs):
         - for EK60 echosounder, allowed parameters include:
           `"sa_correction"`, `"gain_correction"`, `"equivalent_beam_angle"`
         - for AZFP echosounder, allowed parameters include:
-          `"EL"`, `"DS"`, `"TVR"`, `"VTX"`, `"equivalent_beam_angle"`, `"Sv_offset"`
+          `"EL"`, `"DS"`, `"TVR"`, `"VTX0"`, `"equivalent_beam_angle"`, `"Sv_offset"`
 
         Passing in calibration parameters for other echosounders
         are not currently supported.

echopype/calibrate/cal_params.py

@@ -29,7 +29,7 @@
         "impedance_transceiver",  # z_er
         "receiver_sampling_frequency",
     ),
-    "AZFP": ("EL", "DS", "TVR", "VTX", "equivalent_beam_angle", "Sv_offset"),
+    "AZFP": ("EL", "DS", "TVR", "VTX0", "equivalent_beam_angle", "Sv_offset"),
 }
 
 EK80_DEFAULT_PARAMS = {
@@ -352,7 +352,7 @@ def get_cal_params_AZFP(beam: xr.DataArray, vend: xr.DataArray, user_dict: dict)
                 out_dict[p] = beam[p]  # has only channel dim
 
             # Params from Vendor_specific group
-            elif p in ["EL", "DS", "TVR", "VTX", "Sv_offset"]:
+            elif p in ["EL", "DS", "TVR", "VTX0", "Sv_offset"]:
                 out_dict[p] = vend[p]  # these params only have the channel dimension
 
     return out_dict

echopype/calibrate/calibrate_azfp.py

@@ -63,7 +63,7 @@ def _cal_power_samples(self, cal_type, **kwargs):
         # TODO: take care of dividing by zero encountered in log10
         spreading_loss = 20 * np.log10(self.range_meter)
         absorption_loss = 2 * self.env_params["sound_absorption"] * self.range_meter
-        SL = self.cal_params["TVR"] + 20 * np.log10(self.cal_params["VTX"])  # eq.(2)
+        SL = self.cal_params["TVR"] + 20 * np.log10(self.cal_params["VTX0"])  # eq.(2)
 
         # scaling factor (slope) in Fig.G-1, units Volts/dB], see p.84
         a = self.cal_params["DS"]

echopype/calibrate/range.py

@@ -60,7 +60,7 @@ def compute_range_AZFP(echodata: EchoData, env_params: Dict, cal_type: str) -> x
     # Notation below follows p.86 of user manual
     N = vend["number_of_samples_per_average_bin"]  # samples per bin
     f = vend["digitization_rate"]  # digitization rate
-    L = vend["lockout_index"]  # number of lockout samples
+    L = vend["lock_out_index"]  # number of lockout samples
 
     # keep this in ref of AZFP matlab code,
     # set to 1 since we want to calculate from raw data

echopype/convert/parse_azfp.py

@@ -1,5 +1,5 @@
 import os
-import xml.dom.minidom
+import xml.etree.ElementTree as ET
 from collections import defaultdict
 from datetime import datetime as dt
 from struct import unpack
@@ -8,48 +8,11 @@
 import numpy as np
 
 from ..utils.log import _init_logger
+from ..utils.misc import camelcase2snakecase
 from .parse_base import ParseBase
 
 FILENAME_DATETIME_AZFP = "\\w+.01A"
-XML_INT_PARAMS = {
-    "NumFreq": "num_freq",
-    "SerialNumber": "serial_number",
-    "BurstInterval": "burst_interval",
-    "PingsPerBurst": "pings_per_burst",
-    "AverageBurstPings": "average_burst_pings",
-    "SensorsFlag": "sensors_flag",
-}
-XML_FLOAT_PARAMS = [
-    # Temperature coeffs
-    "ka",
-    "kb",
-    "kc",
-    "A",
-    "B",
-    "C",
-    # Tilt coeffs
-    "X_a",
-    "X_b",
-    "X_c",
-    "X_d",
-    "Y_a",
-    "Y_b",
-    "Y_c",
-    "Y_d",
-]
-XML_FREQ_PARAMS = {
-    "RangeSamples": "range_samples",
-    "RangeAveragingSamples": "range_averaging_samples",
-    "DigRate": "dig_rate",
-    "LockOutIndex": "lockout_index",
-    "Gain": "gain",
-    "PulseLen": "pulse_length",
-    "DS": "DS",
-    "EL": "EL",
-    "TVR": "TVR",
-    "VTX0": "VTX",
-    "BP": "BP",
-}
+
 HEADER_FIELDS = (
     ("profile_flag", "u2"),
     ("profile_number", "u2"),
@@ -64,7 +27,7 @@
     ("second", "u2"),  # Second
     ("hundredths", "u2"),  # Hundredths of a second
     ("dig_rate", "u2", 4),  # Digitalization rate for each channel
-    ("lockout_index", "u2", 4),  # Lockout index for each channel
+    ("lock_out_index", "u2", 4),  # Lockout index for each channel
     ("num_bins", "u2", 4),  # Number of bins for each channel
     (
         "range_samples_per_bin",
@@ -88,7 +51,7 @@
     ("num_chan", "u1"),  # 1, 2, 3, or 4
     ("gain", "u1", 4),  # gain channel 1-4
     ("spare_chan", "u1"),  # spare channel
-    ("pulse_length", "u2", 4),  # Pulse length chan 1-4 uS
+    ("pulse_len", "u2", 4),  # Pulse length chan 1-4 uS
     ("board_num", "u2", 4),  # The board the data came from channel 1-4
     ("frequency", "u2", 4),  # frequency for channel 1-4 in kHz
     (
@@ -118,34 +81,41 @@ def __init__(self, file, params, storage_options={}, dgram_zarr_vars={}):
         self.xml_path = params
 
         # Class attributes
-        self.parameters = dict()
+        self.parameters = defaultdict(list)
         self.unpacked_data = defaultdict(list)
         self.sonar_type = "AZFP"
 
     def load_AZFP_xml(self):
-        """Parse XML file to get params for reading AZFP data."""
-        """Parses the AZFP  XML file.
         """
-
-        def get_value_by_tag_name(tag_name, element=0):
-            """Returns the value in an XML tag given the tag name and the number of occurrences."""
-            return px.getElementsByTagName(tag_name)[element].childNodes[0].data
+        Parses the AZFP XML file.
+        """
 
         xmlmap = fsspec.get_mapper(self.xml_path, **self.storage_options)
-        px = xml.dom.minidom.parse(xmlmap.fs.open(xmlmap.root))
-
-        # Retrieve integer parameters from the xml file
-        for old_name, new_name in XML_INT_PARAMS.items():
-            self.parameters[new_name] = int(get_value_by_tag_name(old_name))
-        # Retrieve floating point parameters from the xml file
-        for param in XML_FLOAT_PARAMS:
-            self.parameters[param] = float(get_value_by_tag_name(param))
-        # Retrieve frequency dependent parameters from the xml file
-        for old_name, new_name in XML_FREQ_PARAMS.items():
-            self.parameters[new_name] = [
-                float(get_value_by_tag_name(old_name, ch))
-                for ch in range(self.parameters["num_freq"])
-            ]
+        root = ET.parse(xmlmap.fs.open(xmlmap.root)).getroot()
+
+        for child in root.iter():
+            if len(child.tag) > 3 and not child.tag.startswith("VTX"):
+                camel_case_tag = camelcase2snakecase(child.tag)
+            else:
+                camel_case_tag = child.tag
+            if len(child.attrib) > 0:
+                for key, val in child.attrib.items():
+                    self.parameters[camel_case_tag + "_" + camelcase2snakecase(key)].append(val)
+
+            if all(char == "\n" for char in child.text):
+                continue
+            else:
+                try:
+                    val = int(child.text)
+                except ValueError:
+                    val = float(child.text)
+
+            self.parameters[camel_case_tag].append(val)
+
+        # Handling the case where there is only one value for each parameter
+        for key, val in self.parameters.items():
+            if len(val) == 1:
+                self.parameters[key] = val[0]
 
     def _compute_temperature(self, ping_num, is_valid):
         """
@@ -245,7 +215,6 @@ def _test_valid_params(params):
                 header_chunk = file.read(self.HEADER_SIZE)
                 if header_chunk:
                     header_unpacked = unpack(self.HEADER_FORMAT, header_chunk)
-
                     # Reading will stop if the file contains an unexpected flag
                     if self._split_header(file, header_unpacked):
                         # Appends the actual 'data values' to unpacked_data
@@ -354,12 +323,12 @@ def _split_header(self, raw, header_unpacked):
 
         field_w_freq = (
             "dig_rate",
-            "lockout_index",
+            "lock_out_index",
             "num_bins",
             "range_samples_per_bin",  # fields with num_freq data
             "data_type",
             "gain",
-            "pulse_length",
+            "pulse_len",
             "board_num",
             "frequency",
         )
@@ -417,12 +386,12 @@ def _check_uniqueness(self):
             # fields with num_freq data
             field_w_freq = (
                 "dig_rate",
-                "lockout_index",
+                "lock_out_index",
                 "num_bins",
                 "range_samples_per_bin",
                 "data_type",
                 "gain",
-                "pulse_length",
+                "pulse_len",
                 "board_num",
                 "frequency",
             )
@@ -478,22 +447,22 @@ def _get_ping_time(self):
         self.ping_time = ping_time
 
     @staticmethod
-    def _calc_Sv_offset(f, pulse_length):
+    def _calc_Sv_offset(f, pulse_len):
         """Calculate the compensation factor for Sv calculation."""
         # TODO: this method seems should be in echopype.process
         if f > 38000:
-            if pulse_length == 300:
+            if pulse_len == 300:
                 return 1.1
-            elif pulse_length == 500:
+            elif pulse_len == 500:
                 return 0.8
-            elif pulse_length == 700:
+            elif pulse_len == 700:
                 return 0.5
-            elif pulse_length == 900:
+            elif pulse_len == 900:
                 return 0.3
-            elif pulse_length == 1000:
+            elif pulse_len == 1000:
                 return 0.3
         else:
-            if pulse_length == 500:
+            if pulse_len == 500:
                 return 1.1
-            elif pulse_length == 1000:
+            elif pulse_len == 1000:
                 return 0.7

echopype/convert/set_groups_azfp.py

@@ -81,13 +81,14 @@ def _create_unique_channel_name(self):
         """
 
         serial_number = self.parser_obj.unpacked_data["serial_number"]
+        frequency_number = self.parser_obj.parameters["frequency_number"]
 
         if serial_number.size == 1:
             freq_as_str = self.freq_sorted.astype(int).astype(str)
 
             # TODO: replace str(i+1) with Frequency Number from XML
             channel_id = [
-                str(serial_number) + "-" + freq + "-" + str(i + 1)
+                str(serial_number) + "-" + freq + "-" + frequency_number[i]
                 for i, freq in enumerate(freq_as_str)
             ]
 
@@ -146,8 +147,7 @@ def set_sonar(self) -> xr.Dataset:
             "sonar_model": self.sonar_model,
             "sonar_serial_number": int(self.parser_obj.unpacked_data["serial_number"]),
             "sonar_software_name": "AZFP",
-            # TODO: software version is hardwired. Read it from the XML file's AZFP_Version node
-            "sonar_software_version": "1.4",
+            "sonar_software_version": "based on AZFP Matlab version 1.4",
             "sonar_type": "echosounder",
         }
         ds = ds.assign_attrs(sonar_attr_dict)
@@ -320,7 +320,7 @@ def set_beam(self) -> List[xr.Dataset]:
             del N_tmp
 
         tdn = (
-            unpacked_data["pulse_length"][self.freq_ind_sorted] / 1e6
+            unpacked_data["pulse_len"][self.freq_ind_sorted] / 1e6
         )  # Convert microseconds to seconds
         range_samples_per_bin = unpacked_data["range_samples_per_bin"][
             self.freq_ind_sorted
@@ -500,15 +500,15 @@ def set_vendor(self) -> xr.Dataset:
         unpacked_data = self.parser_obj.unpacked_data
         parameters = self.parser_obj.parameters
         ping_time = self.parser_obj.ping_time
-        tdn = parameters["pulse_length"][self.freq_ind_sorted] / 1e6
+        tdn = parameters["pulse_len"][self.freq_ind_sorted] / 1e6
         anc = np.array(unpacked_data["ancillary"])  # convert to np array for easy slicing
 
         # Build variables in the output xarray Dataset
         Sv_offset = np.zeros_like(self.freq_sorted)
         for ind, ich in enumerate(self.freq_ind_sorted):
             # TODO: should not access the private function, better to compute Sv_offset in parser
             Sv_offset[ind] = self.parser_obj._calc_Sv_offset(
-                self.freq_sorted[ind], unpacked_data["pulse_length"][ich]
+                self.freq_sorted[ind], unpacked_data["pulse_len"][ich]
             )
 
         ds = xr.Dataset(
@@ -532,9 +532,9 @@ def set_vendor(self) -> xr.Dataset:
                         "A/D converter when digitizing the returned acoustic signal"
                     },
                 ),
-                "lockout_index": (
+                "lock_out_index": (
                     ["channel"],
-                    unpacked_data["lockout_index"][self.freq_ind_sorted],
+                    unpacked_data["lock_out_index"][self.freq_ind_sorted],
                     {
                         "long_name": "The distance, rounded to the nearest Bin Size after the "
                         "pulse is transmitted that over which AZFP will ignore echoes"
@@ -648,6 +648,17 @@ def set_vendor(self) -> xr.Dataset:
                     parameters["gain"][self.freq_ind_sorted],
                     {"long_name": "(From XML file) Gain correction"},
                 ),
+                "instrument_type": parameters["instrument_type"][0],
+                "minor": parameters["minor"],
+                "major": parameters["major"],
+                "date": parameters["date"],
+                "program": parameters["program"],
+                "cpu": parameters["cpu"],
+                "serial_number": parameters["serial_number"],
+                "board_version": parameters["board_version"],
+                "file_version": parameters["file_version"],
+                "parameter_version": parameters["parameter_version"],
+                "configuration_version": parameters["configuration_version"],
                 "XML_digitization_rate": (
                     ["channel"],
                     parameters["dig_rate"][self.freq_ind_sorted],
@@ -659,7 +670,7 @@ def set_vendor(self) -> xr.Dataset:
                 ),
                 "XML_lockout_index": (
                     ["channel"],
-                    parameters["lockout_index"][self.freq_ind_sorted],
+                    parameters["lock_out_index"][self.freq_ind_sorted],
                     {
                         "long_name": "(From XML file) The distance, rounded to the nearest "
                         "Bin Size after the pulse is transmitted that over which AZFP will "
@@ -680,10 +691,25 @@ def set_vendor(self) -> xr.Dataset:
                         "units": "dB re 1uPa/V at 1m",
                     },
                 ),
-                "VTX": (
+                "VTX0": (
                     ["channel"],
-                    parameters["VTX"][self.freq_ind_sorted],
-                    {"long_name": "Amplified voltage sent to the transducer"},
+                    parameters["VTX0"][self.freq_ind_sorted],
+                    {"long_name": "Amplified voltage 0 sent to the transducer"},
+                ),
+                "VTX1": (
+                    ["channel"],
+                    parameters["VTX1"][self.freq_ind_sorted],
+                    {"long_name": "Amplified voltage 1 sent to the transducer"},
+                ),
+                "VTX2": (
+                    ["channel"],
+                    parameters["VTX2"][self.freq_ind_sorted],
+                    {"long_name": "Amplified voltage 2 sent to the transducer"},
+                ),
+                "VTX3": (
+                    ["channel"],
+                    parameters["VTX3"][self.freq_ind_sorted],
+                    {"long_name": "Amplified voltage 3 sent to the transducer"},
                 ),
                 "Sv_offset": (["channel"], Sv_offset),
                 "number_of_samples_digitized_per_pings": (