Merge pull request #50 from pysat/file_selection

load simple and multiple file types
pysat · Jun 2, 2021 · 462dd78 · 462dd78
2 parents 1a9def6 + 98b3f9a
commit 462dd78
Show file tree

Hide file tree

Showing 5 changed files with 278 additions and 296 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
   - Added coords from pysat.utils
   - Added Vertical TEC Instrument
   - Added documentation
+  - Added load routine for simple formatted data
+  - Expanded feedback during data downloads
   - Updated documentation configuration to improve maintainability
   - Updated documentation style, displaying logo on sidebar in html format
   - Changed zenodo author name format for better BibTeX compliance
@@ -22,6 +24,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
   - Updated load for multiple files in pandas format
   - Fixed remote listing routine to return filenames instead of experiments
   - Fixed bug introduced by change in xarray requiring engine kwarg
+  - Fixed bug that would not list multiple types of files
 
 ## [0.0.3] - 2020-06-15
 - pypi compatibility

diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py
@@ -56,7 +56,6 @@
 import functools
 import numpy as np
 
-from pysat.instruments.methods import general as ps_gen
 from pysat import logger
 
 from pysatMadrigal.instruments.methods import general, dmsp
@@ -173,6 +172,10 @@ def clean(self):
 #
 # Use the default Madrigal and pysat methods
 
+# Support listing the local files
+list_files = functools.partial(general.list_files,
+                               supported_tags=supported_tags)
+
 # Set the list_remote_files routine
 list_remote_files = functools.partial(general.list_remote_files,
                                       inst_code=madrigal_inst_code,
@@ -183,51 +186,6 @@ def clean(self):
 load = general.load
 
 
-def list_files(tag=None, inst_id=None, data_path=None, format_str=None,
-               supported_tags=supported_tags, delimiter=None, file_type='hdf5'):
-    """Return a Pandas Series of every data file for this Instrument
-
-    Parameters
-    ----------
-    tag : string or NoneType
-        Denotes type of file to load.  Accepted types are <tag strings>.
-        (default=None)
-    inst_id : string or NoneType
-        Specifies the satellite ID for a constellation.  Not used.
-        (default=None)
-    data_path : string or NoneType
-        Path to data directory.  If None is specified, the value previously
-        set in Instrument.files.data_path is used.  (default=None)
-    format_str : string or NoneType
-        User specified file format.  If None is specified, the default
-        formats associated with the supplied tags are used. (default=None)
-    supported_tags : dict or NoneType
-        keys are inst_id, each containing a dict keyed by tag
-        where the values file format template strings. (default=None)
-    delimiter : string
-        Delimiter string upon which files will be split (e.g., '.')
-    file_type : string
-        File format for Madrigal data.  Load routines currently only accepts
-        'hdf5' and 'netCDF4', but any of the Madrigal options may be used
-        here. (default='netCDF4')
-
-    Returns
-    -------
-    out : pysat.Files.from_os : pysat._files.Files
-        A class containing the verified available files
-
-    """
-    if supported_tags[inst_id][tag].find('{file_type}') > 0:
-        supported_tags[inst_id][tag] = supported_tags[inst_id][tag].format(
-            file_type=file_type)
-
-    out = ps_gen.list_files(tag=tag, inst_id=inst_id, data_path=data_path,
-                            format_str=format_str, delimiter=delimiter,
-                            supported_tags=supported_tags)
-
-    return out
-
-
 def download(date_array, tag='', inst_id='', data_path=None, user=None,
              password=None, file_type='hdf5'):
     """Downloads data from Madrigal.
@@ -252,9 +210,7 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None,
     password : string
         Password for data download. (default=None)
     file_type : string
-        File format for Madrigal data.  Load routines currently only accepts
-        'hdf5' and 'netCDF4', but any of the Madrigal options may be used
-        here. (default='hdf5')
+        File format for Madrigal data. (default='hdf5')
 
     Note
     ----
@@ -270,5 +226,5 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None,
     """
     general.download(date_array, inst_code=str(madrigal_inst_code),
                      kindat=madrigal_tag[inst_id][tag], data_path=data_path,
-                     user=user, password=password)
+                     user=user, password=password, file_type=file_type)
     return
diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py
@@ -40,7 +40,6 @@
 import functools
 import numpy as np
 
-from pysat.instruments.methods import general as ps_gen
 from pysat import logger
 
 from pysatMadrigal.instruments.methods import general, gnss
@@ -116,64 +115,18 @@ def clean(self):
 #
 # Use the default Madrigal methods
 
-# support listing files currently available on remote server (Madrigal)
+# Support listing the local files
+list_files = functools.partial(general.list_files,
+                               supported_tags=supported_tags,
+                               two_digit_year_break=99)
+
+# Support listing files currently available on remote server (Madrigal)
 list_remote_files = functools.partial(general.list_remote_files,
                                       supported_tags=remote_tags,
                                       inst_code=madrigal_inst_code,
                                       kindats=madrigal_tag)
 
 
-def list_files(tag=None, inst_id=None, data_path=None, format_str=None,
-               supported_tags=supported_tags, two_digit_year_break=99,
-               delimiter=None, file_type='netCDF4'):
-    """Return a Pandas Series of every data file for this Instrument
-
-    Parameters
-    -----------
-    tag : string or NoneType
-        Denotes type of file to load.  Accepted types are <tag strings>.
-        (default=None)
-    inst_id : string or NoneType
-        Specifies the satellite ID for a constellation.  Not used.
-        (default=None)
-    data_path : string or NoneType
-        Path to data directory.  If None is specified, the value previously
-        set in Instrument.files.data_path is used.  (default=None)
-    format_str : string or NoneType
-        User specified file format.  If None is specified, the default
-        formats associated with the supplied tags are used. (default=None)
-    supported_tags : dict or NoneType
-        keys are inst_id, each containing a dict keyed by tag
-        where the values file format template strings. (default=None)
-    two_digit_year_break : int
-        If filenames only store two digits for the year, then
-        '1900' will be added for years >= two_digit_year_break
-        and '2000' will be added for years < two_digit_year_break.
-    delimiter : string
-        Delimiter string upon which files will be split (e.g., '.')
-    file_type : string
-        File format for Madrigal data.  Load routines currently only accepts
-        'hdf5' and 'netCDF4', but any of the Madrigal options may be used
-        here. (default='netCDF4')
-
-    Returns
-    -------
-    out : pysat.Files.from_os : pysat._files.Files
-        A class containing the verified available files
-
-    """
-    if supported_tags[inst_id][tag].find('{file_type}') > 0:
-        supported_tags[inst_id][tag] = supported_tags[inst_id][tag].format(
-            file_type=file_type)
-
-    out = ps_gen.list_files(tag=tag, inst_id=inst_id, data_path=data_path,
-                            format_str=format_str, delimiter=delimiter,
-                            supported_tags=supported_tags,
-                            two_digit_year_break=two_digit_year_break)
-
-    return out
-
-
 def download(date_array, tag='', inst_id='', data_path=None, user=None,
              password=None, url='http://cedar.openmadrigal.org',
              file_type='netCDF4'):
@@ -184,25 +137,23 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None,
     date_array : array-like
         list of datetimes to download data for. The sequence of dates need not
         be contiguous.
-    tag : string
+    tag : str
         Tag identifier used for particular dataset. This input is provided by
         pysat. (default='')
-    inst_id : string
+    inst_id : str
         Instrument ID string identifier used for particular dataset. This input
         is provided by pysat. (default='')
-    data_path : string
+    data_path : str
         Path to directory to download data to. (default=None)
-    user : string
+    user : str
         User string input used for download. Provided by user and passed via
         pysat. (default=None)
-    password : string
+    password : str
         Password for data download. (default=None)
-    url : string
+    url : str
         URL for Madrigal site (default='http://cedar.openmadrigal.org')
-    file_type : string
-        File format for Madrigal data.  Load routines currently only accepts
-        'hdf5' and 'netCDF4', but any of the Madrigal options may be used
-        here. (default='netCDF4')
+    file_type : str
+        File format for Madrigal data. (default='netCDF4')
 
     Note
     ----
@@ -223,26 +174,22 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None,
     return
 
 
-def load(fnames, tag=None, inst_id=None, file_type='netCDF4'):
+def load(fnames, tag=None, inst_id=None):
     """ Routine to load the GNSS TEC data
 
     Parameters
-    -----------
+    ----------
     fnames : list
         List of filenames
-    tag : string or NoneType
+    tag : str or NoneType
         tag name used to identify particular data set to be loaded.
         This input is nominally provided by pysat itself. (default=None)
-    inst_id : string or NoneType
+    inst_id : str or NoneType
         Instrument ID used to identify particular data set to be loaded.
         This input is nominally provided by pysat itself. (default=None)
-    file_type : string
-        File format for Madrigal data. Currently only accepts 'hdf5' and
-        'netCDF4', but any of the supported Madrigal options may be used here.
-        (default='netCDF4')
 
     Returns
-    --------
+    -------
     data : xarray.Dataset
         Object containing satellite data
     meta : pysat.Meta
@@ -257,9 +204,7 @@ def load(fnames, tag=None, inst_id=None, file_type='netCDF4'):
                                      'sec', 'ut1_unix', 'ut2_unix', 'recno']}}
 
     # Load the specified data
-    data, meta = general.load(fnames, tag, inst_id,
-                              xarray_coords=xcoords[tag],
-                              file_type=file_type)
+    data, meta = general.load(fnames, tag, inst_id, xarray_coords=xcoords[tag])
 
     # Squeeze the kindat and kinst 'coordinates', but keep them as floats
     squeeze_dims = np.array(['kindat', 'kinst'])