Merge pull request #67 from pysat/general_madrigal_inst

ENH: General Madrigal instrument

CHANGELOG.md

@@ -2,11 +2,16 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](https://semver.org/).
 
-## [0.1.0] - 2021-XX-XX
+## [0.1.0] - 2022-XX-XX
 - Enhancements
    - Improved definitions of general and GNSS meta data
    - Removed unused logic in cleaning routines
    - Moved warning for no cleaning of JRO ISR data to preprocess
+   - Added a general Madrigal instrument for time-series data
+   - Added functions to specify all known Madrigal instrument codes and
+     file formats
+   - Adapted general listing functions to allow file formats with '*' wildcards
+     between '.' delimiters, required for some Madrigal file formats
    - Standardized the Instrument method kwarg defaults
    - Added 'site' tag to the GNSS TEC Instrument
    - Added support for varied use of `two_digit_year_break` to 
@@ -17,6 +22,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
    - Improved the docstring style
 - Testing
    - Added unit tests for general, JRO, and GNSS methods
+   - Added the packaging module to handle version logic
 - Bug
    - Fixed bugs in the coordinate conversion functions
 

docs/supported_instruments.rst

@@ -22,7 +22,7 @@ TEC is provided by MIT Haystack.
    :members:
 
 JRO_ISR
-----------------
+-------
 
 The incoherent scatter radar (ISR) at the
 `Jicamarca Radio Observatory <http://jro.igp.gob.pe/english/>`_ regularly
@@ -31,3 +31,22 @@ magnetic equator over Peru.
 
 .. automodule:: pysatMadrigal.instruments.jro_isr
    :members:
+
+
+Madrigal_Pandas
+---------------
+
+A general instrument for Madrigal time-series data.  This
+:py:class:`pysat.Instrument` uses Madrigal instrument codes and kindats to
+support the use of any of the Madrigal time-series data sets.  There are some
+further constraints in that the data set's Madrigal naming convention must be
+parsable by pysat.  Currently nine Madrigal instrument codes are supported by
+this :py:class:`pysat.Instrument`.  When possible, using a specific instrument
+module is recommended, since that instrument module will have additional
+support (e.g., cleaning methods, experiment acknowledgements, and references).
+
+.. automodule:: pysatMadrigal.instruments.madrigal_pandas
+   :members:
+
+
+

pysatMadrigal/instruments/__init__.py

@@ -1,8 +1,11 @@
 # Import Madrigal instruments
-from pysatMadrigal.instruments import dmsp_ivm, gnss_tec, jro_isr
+from pysatMadrigal.instruments import dmsp_ivm
+from pysatMadrigal.instruments import gnss_tec
+from pysatMadrigal.instruments import jro_isr
+from pysatMadrigal.instruments import madrigal_pandas
 
 # Import Madrigal methods
 from pysatMadrigal.instruments import methods  # noqa F401
 
 # Define variable name with all available instruments
-__all__ = ['dmsp_ivm', 'gnss_tec', 'jro_isr']
+__all__ = ['dmsp_ivm', 'gnss_tec', 'jro_isr', 'madrigal_pandas']

pysatMadrigal/instruments/gnss_tec.py

@@ -53,22 +53,21 @@
 
 pandas_format = False
 
+# Madrigal tags
+madrigal_inst_code = 8000
+madrigal_tag = {'': {'vtec': '3500', 'site': '3506'}}
+# TODO(#12): `, 'los': '3505'}}`
+
 # Local attributes
-dname = '{{month:02d}}{{day:02d}}'
-vname = '.{{version:03d}}'
-supported_tags = {ss: {'vtec': ''.join(['gps{{year:02d}}', dname, 'g', vname,
-                                        ".{file_type}"]),
-                       'site': ''.join(['site_{{year:04d}}', dname, vname,
-                                        ".{file_type}"])}
+fname = general.madrigal_file_format_str(madrigal_inst_code,
+                                         verbose=False).split("*")
+supported_tags = {ss: {'vtec': ''.join(['gps', fname[1], 'g', fname[2]]),
+                       'site': ''.join(['site_{{year:04d}}{{month:02d}}',
+                                        '{{day:02d}}', fname[2]])}
                   for ss in inst_ids.keys()}
 remote_tags = {ss: {kk: supported_tags[ss][kk].format(file_type='hdf5')
                     for kk in inst_ids[ss]} for ss in inst_ids.keys()}
 
-# Madrigal tags
-madrigal_inst_code = 8000
-madrigal_tag = {'': {'vtec': '3500', 'site': '3506'}}
-# , 'los': '3505'}} <- Issue #12
-
 # ----------------------------------------------------------------------------
 # Instrument test attributes
 

pysatMadrigal/instruments/jro_isr.py

@@ -1,13 +1,11 @@
+#!/usr/bin/env python
+# Full license can be found in License.md
+# Full author list can be found in .zenodo.json file
+# DOI:10.5281/zenodo.3824979
+# ----------------------------------------------------------------------------
 # -*- coding: utf-8 -*-.
 """Supports the Incoherent Scatter Radar at the Jicamarca Radio Observatory.
 
-The Incoherent Scatter Radar (ISR) at the Jicamarca Radio Observatory (JRO)
-observes ion drifts, line-of-sight neutral winds, electron density and
-temperature, ion temperature, and ion composition through three overarching
-experiments.
-
-Downloads data from the JRO Madrigal Database.
-
 Properties
 ----------
 platform
@@ -30,7 +28,15 @@
 
 Note
 ----
-    Please provide name and email when downloading data with this routine.
+The Incoherent Scatter Radar (ISR) at the Jicamarca Radio Observatory (JRO)
+observes ion drifts, line-of-sight neutral winds, electron density and
+temperature, ion temperature, and ion composition through three overarching
+experiments.
+
+Downloads data from the JRO Madrigal Database.
+
+Please provide name (user) and email (password) when downloading data with this
+routine.
 
 """
 
@@ -55,24 +61,23 @@
 
 pandas_format = False
 
-# Local attributes
-jro_fname1 = 'jro{{year:4d}}{{month:02d}}{{day:02d}}'
-jro_fname2 = '.{{version:03d}}.{file_type}'
-supported_tags = {ss: {'drifts': jro_fname1 + "drifts" + jro_fname2,
-                       'drifts_ave': jro_fname1 + "drifts_avg" + jro_fname2,
-                       'oblique_stan': jro_fname1 + jro_fname2,
-                       'oblique_rand': jro_fname1 + "?" + jro_fname2,
-                       'oblique_long': jro_fname1 + "?" + jro_fname2}
-                  for ss in inst_ids.keys()}
-remote_tags = {ss: {kk: supported_tags[ss][kk].format(file_type='hdf5')
-                    for kk in inst_ids[ss]} for ss in inst_ids.keys()}
-
 # Madrigal tags
 madrigal_inst_code = 10
 madrigal_tag = {'': {'drifts': "1910", 'drifts_ave': "1911",
                      'oblique_stan': "1800", 'oblique_rand': "1801",
                      'oblique_long': "1802"}, }
 
+# Local attributes
+jro_fname = general.madrigal_file_format_str(madrigal_inst_code, verbose=False)
+supported_tags = {ss: {'drifts': jro_fname.replace("*", "drifts"),
+                       'drifts_ave': jro_fname.replace("*", "drifts_avg"),
+                       'oblique_stan': jro_fname.replace("*", ""),
+                       'oblique_rand': jro_fname.replace("*", "?"),
+                       'oblique_long': jro_fname.replace("*", "?")}
+                  for ss in inst_ids.keys()}
+remote_tags = {ss: {kk: supported_tags[ss][kk].format(file_type='hdf5')
+                    for kk in inst_ids[ss]} for ss in inst_ids.keys()}
+
 # ----------------------------------------------------------------------------
 # Instrument test attributes
 
@@ -202,7 +207,7 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None,
     Notes
     -----
     The user's names should be provided in field user. Ruby Payne-Scott should
-    be entered as Ruby+Payne-Scott
+    be entered as "Ruby Payne-Scott"
 
     The password field should be the user's email address. These parameters
     are passed to Madrigal when downloading.
@@ -214,6 +219,7 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None,
     general.download(date_array, inst_code=str(madrigal_inst_code),
                      kindat=madrigal_tag[inst_id][tag], data_path=data_path,
                      user=user, password=password, file_type=file_type)
+    return
 
 
 def load(fnames, tag='', inst_id=''):