Merge pull request #160 from pysat/enh/cdf_xarray

BUG/ENH: cdf support for multi_file_day xarray, TIMED SABER
pysat · Apr 12, 2023 · 7ccc4bf · 7ccc4bf
2 parents bb66ff3 + f47edb1
commit 7ccc4bf
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   * Updated CDAWeb routines to allow for data stored by year/day-of-year
   * Updated GOLD nmax to sort scans by time.
   * Added 1 usec to GOLD nmax channel B times to ensure uniqueness
+  * Fixed multi-file loads for cdf xarray datasets.
 * Documentation
   * Added TIMED-GUVI platform
   * Added missing sub-module imports
@@ -31,7 +32,9 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   * Updated platform methods to follow a consistent style and work with the
     general `init` function
   * Added unit tests for the different platform method attributes
-  * xarray support for TIMED SEE
+  * xarray support for TIMED SABER and SEE
+  * Added `drop_dims` kwarg to `load_xarray` interface so that orphan dims can
+    be removed before attempting to merge.
 * Deprecations
   * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps
 * Maintenance

diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py
@@ -64,8 +64,8 @@ def try_inst_dict(inst_id, tag, supported_tags):
 
 def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1),
          flatten_twod=True, pandas_format=True, epoch_name='Epoch',
-         meta_processor=None, meta_translation=None, drop_meta_labels=None,
-         use_cdflib=None):
+         drop_dims=None, meta_processor=None, meta_translation=None,
+         drop_meta_labels=None, use_cdflib=None):
     """Load NASA CDAWeb CDF files.
 
     Parameters
@@ -93,6 +93,10 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1),
         specified by `epoch_origin` with units specified by `epoch_unit`. This
         epoch variable will be converted to a `DatetimeIndex` for consistency
         across pysat instruments.  (default='Epoch')
+    drop_dims : list or NoneType
+        List of variable dimensions that should be dropped. Applied
+        to data as loaded from the file. Used only from xarray Dataset.
+        (default=None)
     meta_processor : function or NoneType
         If not None, a dict containing all of the loaded metadata will be
         passed to `meta_processor` which should return a filtered version
@@ -143,6 +147,7 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1),
 
         data, meta = load_xarray(fnames, tag=tag, inst_id=inst_id,
                                  epoch_name=epoch_name,
+                                 drop_dims=drop_dims,
                                  file_cadence=file_cadence,
                                  meta_processor=meta_processor,
                                  meta_translation=meta_translation,
@@ -264,7 +269,7 @@ def load_xarray(fnames, tag='', inst_id='',
                         'min_val': ('ValidMin', float),
                         'max_val': ('ValidMax', float),
                         'fill_val': ('FillVal', float)},
-                epoch_name='Epoch', meta_processor=None,
+                epoch_name='Epoch', drop_dims=None, meta_processor=None,
                 meta_translation=None, drop_meta_labels=None):
     """Load NASA CDAWeb CDF files into an xarray Dataset.
 
@@ -295,6 +300,9 @@ def load_xarray(fnames, tag='', inst_id='',
         specified by `epoch_origin` with units specified by `epoch_unit`. This
         epoch variable will be converted to a `DatetimeIndex` for consistency
         across pysat instruments.  (default='Epoch')
+    drop_dims : list or NoneType
+        List of variable dimensions that should be dropped. Applied
+        to data as loaded from the file. (default=None)
     meta_processor : function or NoneType
         If not None, a dict containing all of the loaded metadata will be
         passed to `meta_processor` which should return a filtered version
@@ -357,11 +365,13 @@ def load_xarray(fnames, tag='', inst_id='',
 
         for lfname in lfnames:
             temp_data = cdflib.cdf_to_xarray(lfname, to_datetime=True)
+            if drop_dims:
+                temp_data = temp_data.drop_dims(drop_dims)
             ldata.append(temp_data)
 
-        # Combine individual files together
+        # Combine individual files together, concat along epoch
         if len(ldata) > 0:
-            data = xr.combine_by_coords(ldata)
+            data = xr.combine_nested(ldata, epoch_name)
 
     all_vars = io.xarray_all_vars(data)
 

diff --git a/pysatNASA/instruments/timed_saber.py b/pysatNASA/instruments/timed_saber.py
@@ -68,7 +68,7 @@
 
 # Set to False to specify using xarray (not using pandas)
 # Set to True if data will be returned via a pandas DataFrame
-pandas_format = True
+pandas_format = False
 
 # ----------------------------------------------------------------------------
 # Instrument test attributes
@@ -99,7 +99,8 @@
                                supported_tags=supported_tags)
 
 # Set the load routine
-load = cdw.load
+load = functools.partial(cdw.load, pandas_format=pandas_format,
+                         drop_dims='record0')
 
 # Set the download routine
 download_tags = {'': {'': 'TIMED_L2A_SABER'}}