Pseudograms remix

gutils/__init__.py

@@ -14,6 +14,11 @@
 import pandas as pd
 from scipy.signal import boxcar, convolve
 
+from pocean.meta import MetaInterface
+from pocean.utils import (
+    dict_update
+)
+
 import logging
 L = logging.getLogger(__name__)
 
@@ -238,6 +243,48 @@ def get_profile_data(profile, method=None):
     return tuv(t=t, x=x, y=y)
 
 
+def read_attrs(config_path=None, template=None):
+
+    def cfg_file(name):
+        return os.path.join(
+            config_path,
+            name
+        )
+
+    template = template or 'trajectory'
+
+    if os.path.isfile(template):
+        default_attrs_path = template
+    else:
+        template_dir = os.path.join(os.path.dirname(__file__), 'templates')
+        default_attrs_path = os.path.join(template_dir, '{}.json'.format(template))
+        if not os.path.isfile(default_attrs_path):
+            L.error("Template path {} not found, using defaults.".format(default_attrs_path))
+            default_attrs_path = os.path.join(template_dir, 'trajectory.json')
+
+    # Load in template defaults
+    defaults = dict(MetaInterface.from_jsonfile(default_attrs_path))
+
+    # Load instruments
+    ins = {}
+    if config_path:
+        ins_attrs_path = cfg_file("instruments.json")
+        if os.path.isfile(ins_attrs_path):
+            ins = dict(MetaInterface.from_jsonfile(ins_attrs_path))
+
+    # Load deployment attributes (including some global attributes)
+    deps = {}
+    if config_path:
+        deps_attrs_path = cfg_file("deployment.json")
+        if os.path.isfile(deps_attrs_path):
+            deps = dict(MetaInterface.from_jsonfile(deps_attrs_path))
+
+    # Update, highest precedence updates last
+    one = dict_update(defaults, ins)
+    two = dict_update(one, deps)
+    return two
+
+
 def safe_makedirs(folder):
     try:
         os.makedirs(folder)

gutils/filters.py

@@ -2,6 +2,7 @@
 # coding=utf-8
 import os
 import pandas as pd
+import numpy as np
 
 from gutils.yo import assign_profiles
 
@@ -115,16 +116,15 @@ def process_dataset(file,
     try:
         reader = reader_class(file)
         data = reader.standardize(z_axis_method=z_axis_method)
-        # For whatever reason, only the first downward leg of
-        # profile 1 is written to the netCDF file.  The acoustic
-        # sensors for Gretel(unit_507) are on during the upward
-        # portion of the leg.
-        # The data section is also passed to extras processing
-        # for the pseudogram.  If the pseudogram is found, the
-        # echometrics (echodroid) variable are removed from the
-        # data section and reassigned with the extras dimension.
+
+        # The data section is also passed to extras processing.
+        # In certain circumstances, the dimension for certain
+        # variables are reassigned with the extras dimension.
         extras, data = reader.extras(data, **extra_kwargs)
 
+        pseudograms_attrs = extra_kwargs.pop('pseudograms', {})
+        have_pseudograms = pseudograms_attrs.pop('enable', False)
+
         if 'z' not in data.columns:
             L.warning("No Z axis found - Skipping {}".format(file))
             return None, None, None
@@ -184,6 +184,20 @@ def process_dataset(file,
             except BaseException as e:
                 L.error(f"Could not merge 'extras' data, skipping: {e}")
 
+            if have_pseudograms:
+                # To have consistent netCDF files, empty dataframes have to exist
+                # in extras for each valid profile in filtered variable
+                profile_list = set(filtered['profile'].unique())
+                extras_list = set(extras['profile'].unique().astype('int'))
+                profiles_to_add = profile_list.difference(extras_list)
+                first_t_in_profiles = filtered.groupby(by=["profile"]).min()['t']
+                for profile_to_add in profiles_to_add:
+                    empty_df = pd.DataFrame([[np.nan] * len(extras.columns)], columns=extras.columns)
+                    empty_df['profile'] = float(profile_to_add)
+                    empty_df['pseudogram_time'] = first_t_in_profiles[profile_to_add]
+                    empty_df.set_index('pseudogram_time', inplace=True)
+                    extras = empty_df.append(extras)
+
     except ValueError as e:
         L.exception('{} - Skipping'.format(e))
         raise

gutils/nc.py

@@ -24,13 +24,12 @@
     create_ncvar_from_series,
     get_ncdata_from_series
 )
-from pocean.meta import MetaInterface
 from pocean.dsg import (
     IncompleteMultidimensionalTrajectory,
     ContiguousRaggedTrajectoryProfile
 )
 
-from gutils import get_uv_data, get_profile_data, safe_makedirs, setup_cli_logger
+from gutils import get_uv_data, get_profile_data, read_attrs, safe_makedirs, setup_cli_logger
 from gutils.filters import process_dataset
 from gutils.slocum import SlocumReader
 
@@ -47,48 +46,6 @@ class ProfileIdTypes(object):
     FRAME = 3  # "profile" column from the input dataframe
 
 
-def read_attrs(config_path=None, template=None):
-
-    def cfg_file(name):
-        return os.path.join(
-            config_path,
-            name
-        )
-
-    template = template or 'trajectory'
-
-    if os.path.isfile(template):
-        default_attrs_path = template
-    else:
-        template_dir = os.path.join(os.path.dirname(__file__), 'templates')
-        default_attrs_path = os.path.join(template_dir, '{}.json'.format(template))
-        if not os.path.isfile(default_attrs_path):
-            L.error("Template path {} not found, using defaults.".format(default_attrs_path))
-            default_attrs_path = os.path.join(template_dir, 'trajectory.json')
-
-    # Load in template defaults
-    defaults = dict(MetaInterface.from_jsonfile(default_attrs_path))
-
-    # Load instruments
-    ins = {}
-    if config_path:
-        ins_attrs_path = cfg_file("instruments.json")
-        if os.path.isfile(ins_attrs_path):
-            ins = dict(MetaInterface.from_jsonfile(ins_attrs_path))
-
-    # Load deployment attributes (including some global attributes)
-    deps = {}
-    if config_path:
-        deps_attrs_path = cfg_file("deployment.json")
-        if os.path.isfile(deps_attrs_path):
-            deps = dict(MetaInterface.from_jsonfile(deps_attrs_path))
-
-    # Update, highest precedence updates last
-    one = dict_update(defaults, ins)
-    two = dict_update(one, deps)
-    return two
-
-
 def set_scalar_value(value, ncvar):
     if value is None or math.isnan(value):
         ncvar[:] = get_fill_value(ncvar)