Merge branch 'develop' into sty/pyproject

CHANGELOG.md

@@ -11,6 +11,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   * DE2 Fabry-Perot Interferometer (FPI)
   * DE2 Vector Electric Field Instrument (VEFI) and magnetometer
   * DMSP SSUSI EDR-Aurora data
+  * IGS GPS (TEC and ROTI)
   * TIMED GUVI
 * Add TIMED GUVI platform to support L1C intensity datasets.
   * Type of sensor source handled by inst_id with options of
@@ -22,6 +23,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   * Updated CDAWeb routines to allow for data stored by year/day-of-year
   * Updated GOLD nmax to sort scans by time.
   * Added 1 usec to GOLD nmax channel B times to ensure uniqueness
+  * Fixed multi-file loads for cdf xarray datasets.
 * Documentation
   * Added TIMED-GUVI platform
   * Added missing sub-module imports
@@ -30,7 +32,11 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   * Updated platform methods to follow a consistent style and work with the
     general `init` function
   * Added unit tests for the different platform method attributes
-  * xarray support for TIMED SEE
+  * xarray support for TIMED SABER and SEE
+  * Added `drop_dims` kwarg to `load_xarray` interface so that orphan dims can
+    be removed before attempting to merge.
+* Deprecations
+  * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps
 * Maintenance
   * Removed duplicate tests if pysatCDF not isntalled
   * Only test pysatCDF on GitHub Actions for older numpy versions
@@ -39,6 +45,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   * Updated dosctring style for consistency
   * Removed version cap for xarray
   * Added manual workflow to check that latest RC is installable through test pip
+  * Update meta label type for instruments
   * Use pyproject.toml to manage setup
 
 ## [0.0.4] - 2022-11-07

docs/supported_instruments.rst

@@ -154,6 +154,14 @@ ICON MIGHTI
 .. automodule:: pysatNASA.instruments.icon_mighti
    :members:
 
+.. _igs_gps:
+
+IGS GPS
+-------
+
+.. automodule:: pysatNASA.instruments.igs_gps
+   :members:
+
 .. _iss_fpmu:
 
 ISS FPMU

pysatNASA/instruments/__init__.py

@@ -11,7 +11,7 @@
            'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_vefi', 'de2_wats',
            'dmsp_ssusi', 'formosat1_ivm',
            'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti',
-           'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold',
+           'igs_gps', 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold',
            'timed_guvi', 'timed_saber', 'timed_see']
 
 for inst in __all__:

pysatNASA/instruments/icon_euv.py

@@ -212,8 +212,9 @@ def load(fnames, tag='', inst_id='', keep_original_names=False):
     """
     labels = {'units': ('Units', str), 'name': ('Long_Name', str),
               'notes': ('Var_Notes', str), 'desc': ('CatDesc', str),
-              'min_val': ('ValidMin', float),
-              'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)}
+              'min_val': ('ValidMin', (int, float)),
+              'max_val': ('ValidMax', (int, float)),
+              'fill_val': ('FillVal', (int, float))}
 
     meta_translation = {'FieldNam': 'plot'}
 

pysatNASA/instruments/icon_fuv.py

@@ -208,8 +208,9 @@ def load(fnames, tag='', inst_id='', keep_original_names=False):
     """
     labels = {'units': ('Units', str), 'name': ('Long_Name', str),
               'notes': ('Var_Notes', str), 'desc': ('CatDesc', str),
-              'min_val': ('ValidMin', float),
-              'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)}
+              'min_val': ('ValidMin', (int, float)),
+              'max_val': ('ValidMax', (int, float)),
+              'fill_val': ('FillVal', (int, float))}
 
     meta_translation = {'FieldNam': 'plot', 'LablAxis': 'axis',
                         'FIELDNAM': 'plot', 'LABLAXIS': 'axis',

pysatNASA/instruments/icon_ivm.py

@@ -272,8 +272,9 @@ def load(fnames, tag='', inst_id='', keep_original_names=False):
 
     labels = {'units': ('Units', str), 'name': ('Long_Name', str),
               'notes': ('Var_Notes', str), 'desc': ('CatDesc', str),
-              'min_val': ('ValidMin', float),
-              'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)}
+              'min_val': ('ValidMin', (int, float)),
+              'max_val': ('ValidMax', (int, float)),
+              'fill_val': ('FillVal', (int, float))}
 
     meta_translation = {'FieldNam': 'plot', 'LablAxis': 'axis',
                         'ScaleTyp': 'scale',

pysatNASA/instruments/icon_mighti.py

@@ -318,8 +318,9 @@ def load(fnames, tag='', inst_id='', keep_original_names=False):
     """
     labels = {'units': ('Units', str), 'name': ('Long_Name', str),
               'notes': ('Var_Notes', str), 'desc': ('CatDesc', str),
-              'min_val': ('ValidMin', float),
-              'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)}
+              'min_val': ('ValidMin', (int, float)),
+              'max_val': ('ValidMax', (int, float)),
+              'fill_val': ('FillVal', (int, float))}
 
     meta_translation = {'FieldNam': 'plot', 'LablAxis': 'axis',
                         'FIELDNAM': 'plot', 'LABLAXIS': 'axis',

pysatNASA/instruments/igs_gps.py

@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+"""Module for the IGS GPS data products.
+
+Supports GPS data produced from International GNSS Service Total Electron
+Content (TEC).
+
+From CDAWeb (modified):
+
+This directory contains the GPS Total Electron Content (TEC) data produced by
+the International Global Navigation Satellite Systems Service (IGS) Ionosphere
+Working Group and by the Analysis Centers that have contributed to the IGS data
+including CODE of the University of Bern (Switzerland), ESA of the European
+Space Operations Center (ESOC) in Darmstadt (Germany), JPL of the Jet Propulsion
+Laboratory, Pasadena (USA), and UPC of the University Politechnical Catalonia in
+Barcelona (Spain). The IGS data are a computed as a weighted mean of the data
+from the four analysis centers.
+
+The rate of TEC index (ROTI) characterizes TEC fluctuations observed along
+receiver-to-satellite line of sight links over a 5-minute interval.
+The measurement is obtained by processing GNSS dual-frequency phase data and
+computing the standard deviation of the rate of TEC change over that interval
+after removing its background variation trend.
+
+ROTI data are provided as global maps using a 2.5 x 5 degree (geographic
+latitude x longitude) grid. The median ROTI value is calculated in each bin.
+GNSS data contributing to the ROTI computation are primarily collected from
+the global network of International GNSS Service and the regional network of
+Continuous Operating Reference Station (CORS).
+
+Properties
+----------
+platform
+    'igs'
+name
+    'gps'
+tag
+    ['tec', 'roti']
+inst_id
+    ['15min', '1hr', '2hr']
+
+
+Warnings
+--------
+- The cleaning parameters for the instrument are still under development.
+
+
+References
+----------
+M. Hernández-Pajares, J.M. Juan, J. Sanz, R. Orus, A. Garcia-Rigo, J. Feltens,
+A. Komjathy, S.C. Schaer, and A. Krankowski, The IGS VTEC maps: a reliable
+source of ionospheric information since 1998Journal of Geodesy (2009) 83:263–275
+doi:10.1007/s00190-008-0266-1
+
+Feltens, J., M. Angling, N. Jackson‐Booth, N. Jakowski, M. Hoque, M.
+Hernández‐Pajares, A. Aragón‐Àngel, R. Orús, and R. Zandbergen (2011),
+Comparative testing of four ionospheric models driven with GPS measurements,
+Radio Sci., 46, RS0D12,	doi:10.1029/2010RS004584
+
+Peng Chen, Hang Liu, Yongchao Ma, Naiquan Zheng, Accuracy and consistency of
+different global ionospheric maps released by IGS ionosphere associate analysis
+centers, Advances in Space Research, Volume 65, Issue 1, 2020, Pages 163-174,
+doi:10.1016/j.asr.2019.09.042.
+
+"""
+
+import datetime as dt
+import functools
+
+from pysat.instruments.methods import general as mm_gen
+
+from pysatNASA.instruments.methods import cdaweb as cdw
+from pysatNASA.instruments.methods import general as mm_nasa
+from pysatNASA.instruments.methods import igs as mm_igs
+
+# ----------------------------------------------------------------------------
+# Instrument attributes
+
+platform = 'igs'
+name = 'gps'
+tags = {'tec': 'Total Electron Content',
+        'roti': 'Rate of Change in TEC'}
+# tags = {'15min': '15 min cadence TEC',
+#         '1hr': '1 hour cadence TEC',
+#         '2hr': '2 hour cadence TEC'}
+inst_ids = {'15min': ['tec', 'roti'],
+            '1hr': ['tec'],
+            '2hr': ['tec']}
+pandas_format = False
+# ----------------------------------------------------------------------------
+# Instrument test attributes
+
+_test_dates = {jj: {kk: dt.datetime(2013, 1, 1) for kk in inst_ids[jj]}
+               for jj in inst_ids.keys()}
+# ----------------------------------------------------------------------------
+# Instrument methods
+
+
+# Use standard init routine
+init = functools.partial(mm_nasa.init, module=mm_igs, name=name)
+
+
+# No cleaning, use standard warning function instead
+clean = mm_nasa.clean_warn
+
+# ----------------------------------------------------------------------------
+# Instrument functions
+#
+# Use the default CDAWeb and pysat methods
+
+# Set the list_files routine
+cdas_labels = {'15min': {'tec': 'GPS_TEC15MIN_IGS',
+                         'roti': 'GPS_ROTI15MIN_JPL'},
+               '1hr': {'tec': 'GPS_TEC1HR_IGS'},
+               '2hr': {'tec': 'GPS_TEC2HR_IGS'}}
+
+date_ver = '{year:4d}{month:02d}{day:02d}_v{version:02d}'
+fname = '{cdas:s}_{date_ver:s}.cdf'
+
+supported_tags = {id: {tag: fname.format(cdas=cdas_labels[id][tag].lower(),
+                                         date_ver=date_ver)
+                       for tag in inst_ids[id]} for id in inst_ids.keys()}
+list_files = functools.partial(mm_gen.list_files,
+                               supported_tags=supported_tags)
+
+# Set the load routine
+load = functools.partial(cdw.load, pandas_format=pandas_format)
+
+# Set the download routine
+download = functools.partial(cdw.cdas_download, supported_tags=cdas_labels)
+
+# Set the list_remote_files routine
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
+                                      supported_tags=cdas_labels)

pysatNASA/instruments/jpl_gps.py

@@ -1,6 +1,11 @@
 # -*- coding: utf-8 -*-
 """Module for the JPL GPS data products.
 
+.. deprecated:: 0.0.5
+    This module is now included in igs_gps.py.
+    This instrument will be removed in 0.1.0+ to reduce redundancy.
+
+
 Supports ROTI data produced at JPL from International
 GNSS Service Total Electron Content (TEC)
 
@@ -48,6 +53,7 @@
 
 import datetime as dt
 import functools
+import warnings
 
 import pysat
 from pysat.instruments.methods import general as mm_gen
@@ -80,6 +86,10 @@ def init(self):
 
     """
 
+    warnings.warn(" ".join(["The instrument module `jpl_gps` has",
+                            "been deprecated and will be removed in 0.1.0+."]),
+                  DeprecationWarning, stacklevel=2)
+
     pysat.logger.info('')
     self.acknowledgements = mm_gps.ackn_str
     self.references = '\n'.join((mm_gps.refs['mission'],
@@ -106,11 +116,9 @@ def init(self):
 load = functools.partial(cdw.load, pandas_format=pandas_format)
 
 # Set the download routine
-basic_tag = {'remote_dir': '/pub/data/gps/roti15min_jpl/{year:4d}/',
-             'fname': fname}
-download_tags = {'': {'roti': basic_tag}}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'': {'roti': 'GPS_ROTI15MIN_JPL'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)

pysatNASA/instruments/methods/_cdf.py

@@ -360,9 +360,9 @@ def load_variables(self):
     def to_pysat(self, flatten_twod=True,
                  labels={'units': ('Units', str), 'name': ('Long_Name', str),
                          'notes': ('Var_Notes', str), 'desc': ('CatDesc', str),
-                         'min_val': ('ValidMin', float),
-                         'max_val': ('ValidMax', float),
-                         'fill_val': ('FillVal', float)}):
+                         'min_val': ('ValidMin', (int, float)),
+                         'max_val': ('ValidMax', (int, float)),
+                         'fill_val': ('FillVal', (int, float))}):
         """Export loaded CDF data into data, meta for pysat module.
 
         Parameters
@@ -382,9 +382,9 @@ def to_pysat(self, flatten_twod=True,
             that order.
             (default={'units': ('units', str), 'name': ('long_name', str),
                       'notes': ('notes', str), 'desc': ('desc', str),
-                      'min_val': ('value_min', float),
-                      'max_val': ('value_max', float)
-                      'fill_val': ('fill', float)})
+                      'min_val': ('value_min', (int, float)),
+                      'max_val': ('value_max', (int, float))
+                      'fill_val': ('fill', (int, float))})
 
         Returns
         -------

pysatNASA/instruments/methods/cdaweb.py

@@ -64,8 +64,8 @@ def try_inst_dict(inst_id, tag, supported_tags):
 
 def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1),
          flatten_twod=True, pandas_format=True, epoch_name='Epoch',
-         meta_processor=None, meta_translation=None, drop_meta_labels=None,
-         use_cdflib=None):
+         drop_dims=None, meta_processor=None, meta_translation=None,
+         drop_meta_labels=None, use_cdflib=None):
     """Load NASA CDAWeb CDF files.
 
     Parameters
@@ -93,6 +93,10 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1),
         specified by `epoch_origin` with units specified by `epoch_unit`. This
         epoch variable will be converted to a `DatetimeIndex` for consistency
         across pysat instruments.  (default='Epoch')
+    drop_dims : list or NoneType
+        List of variable dimensions that should be dropped. Applied
+        to data as loaded from the file. Used only from xarray Dataset.
+        (default=None)
     meta_processor : function or NoneType
         If not None, a dict containing all of the loaded metadata will be
         passed to `meta_processor` which should return a filtered version
@@ -143,6 +147,7 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1),
 
         data, meta = load_xarray(fnames, tag=tag, inst_id=inst_id,
                                  epoch_name=epoch_name,
+                                 drop_dims=drop_dims,
                                  file_cadence=file_cadence,
                                  meta_processor=meta_processor,
                                  meta_translation=meta_translation,
@@ -264,7 +269,7 @@ def load_xarray(fnames, tag='', inst_id='',
                         'min_val': ('ValidMin', float),
                         'max_val': ('ValidMax', float),
                         'fill_val': ('FillVal', float)},
-                epoch_name='Epoch', meta_processor=None,
+                epoch_name='Epoch', drop_dims=None, meta_processor=None,
                 meta_translation=None, drop_meta_labels=None):
     """Load NASA CDAWeb CDF files into an xarray Dataset.
 
@@ -295,6 +300,9 @@ def load_xarray(fnames, tag='', inst_id='',
         specified by `epoch_origin` with units specified by `epoch_unit`. This
         epoch variable will be converted to a `DatetimeIndex` for consistency
         across pysat instruments.  (default='Epoch')
+    drop_dims : list or NoneType
+        List of variable dimensions that should be dropped. Applied
+        to data as loaded from the file. (default=None)
     meta_processor : function or NoneType
         If not None, a dict containing all of the loaded metadata will be
         passed to `meta_processor` which should return a filtered version
@@ -357,11 +365,13 @@ def load_xarray(fnames, tag='', inst_id='',
 
         for lfname in lfnames:
             temp_data = cdflib.cdf_to_xarray(lfname, to_datetime=True)
+            if drop_dims:
+                temp_data = temp_data.drop_dims(drop_dims)
             ldata.append(temp_data)
 
-        # Combine individual files together
+        # Combine individual files together, concat along epoch
         if len(ldata) > 0:
-            data = xr.combine_by_coords(ldata)
+            data = xr.combine_nested(ldata, epoch_name)
 
     all_vars = io.xarray_all_vars(data)
 

pysatNASA/instruments/methods/general.py

@@ -31,10 +31,16 @@ def init(self, module, name):
 
     # Set references
     refs = getattr(module, 'refs')
+    try:
+        # See if there is a tag level reference
+        inst_refs = refs[name][self.tag]
+    except TypeError:
+        # No tag-level ref, use name-levele
+        inst_refs = refs[name]
     if 'mission' in refs.keys():
-        self.references = '\n'.join((refs['mission'], refs[name]))
+        self.references = '\n'.join((refs['mission'], inst_refs))
     else:
-        self.references = refs[name]
+        self.references = inst_refs
 
     return