Merge pull request #145 from pysat/cdasws

Cdasws used to list remote files when possible
pysat · Mar 30, 2023 · 5e404ee · 5e404ee
2 parents d32dc63 + 1a57834
commit 5e404ee
Show file tree

Hide file tree

Showing 27 changed files with 332 additions and 205 deletions.
diff --git a/.gitignore b/.gitignore
@@ -70,3 +70,6 @@ custom_lint.sh
 
 # IDEs
 .idea/
+
+# vscode
+.vscode
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,14 +8,14 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   * ACE MAG
   * ACE SIS
   * ACE SWEPAM
-* New Instruments
   * TIMED GUVI
   * DMSP SSUSI EDR-Aurora data
 * Add TIMED GUVI platform to support L1C intensity datasets.
   * Type of sensor source handled by inst_id with options of
     spectrograph, imaging
   * Resolution of dataset handled by tag with
     low, high
+* Added CDAWeb methods that can use cdasws to get the remote file list
 * Bug Fixes
   * Updated CDAWeb routines to allow for data stored by year/day-of-year
 * Documentation

diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py
@@ -95,13 +95,12 @@
                          meta_translation=meta_translation)
 
 # Set the download routine
-remote_dir = '/pub/data/ace/epam/level_2_cdaweb/epm_{sid:s}/{{year:4d}}/'
-download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]),
-                            'fname': fname.format(sid=strid[id][tag])}
-                      for tag in inst_ids[id]}
-                 for id in inst_ids.keys()}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'12sec': {'base': 'AC_H3_EPM'},
+                 '5min': {'base': 'AC_H1_EPM', 'key': 'AC_K0_EPM'},
+                 '1hr': {'base': 'AC_H2_EPM', 'key': 'AC_K1_EPM'}}
+
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py
@@ -97,13 +97,14 @@
                          meta_translation=meta_translation)
 
 # Set the download routine
-remote_dir = '/pub/data/ace/mag/level_2_cdaweb/mfi_{sid:s}/{{year:4d}}/'
-download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]),
-                            'fname': fname.format(sid=strid[id][tag])}
-                      for tag in inst_ids[id]}
-                 for id in inst_ids.keys()}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'1sec': {'base': 'AC_H3_MFI'},
+                 '16sec': {'base': 'AC_H0_MFI', 'key': 'AC_K1_MFI'},
+                 '4min': {'base': 'AC_H1_MFI'},
+                 '5min': {'key': 'AC_K0_MFI'},
+                 '1hr': {'base': 'AC_H2_MFI', 'key': 'AC_K2_MFI'}}
+
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py
@@ -90,13 +90,11 @@
                          meta_translation=meta_translation)
 
 # Set the download routine
-remote_dir = '/pub/data/ace/sis/level_2_cdaweb/sis_{sid:s}/{{year:4d}}/'
-download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]),
-                            'fname': fname.format(sid=strid[id][tag])}
-                      for tag in inst_ids[id]}
-                 for id in inst_ids.keys()}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'256sec': {'base': 'AC_H1_SIS'},
+                 '1hr': {'base': 'AC_H2_SIS', 'key': 'AC_K0_SIS'}}
+
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py
@@ -93,13 +93,12 @@
                          meta_translation=meta_translation)
 
 # Set the download routine
-remote_dir = '/pub/data/ace/swepam/level_2_cdaweb/swe_{sid:s}/{{year:4d}}/'
-download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]),
-                            'fname': fname.format(sid=strid[id][tag])}
-                      for tag in inst_ids[id]}
-                 for id in inst_ids.keys()}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'64sec': {'base': 'AC_H0_SWE'},
+                 '5min': {'key': 'AC_K0_SWE'},
+                 '1hr': {'base': 'AC_H2_SWE', 'key': 'AC_K1_SWE'}}
+
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/cnofs_ivm.py b/pysatNASA/instruments/cnofs_ivm.py
@@ -247,12 +247,9 @@ def clean(self):
 # Set the load routine
 load = cdw.load
 
-# Set the download routine
-basic_tag = {'remote_dir': '/pub/data/cnofs/cindi/ivm_500ms_cdf/{year:4d}/',
-             'fname': fname}
-download_tags = {'': {'': basic_tag}}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'': {'': 'CNOFS_CINDI_IVM_500MS'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/cnofs_plp.py b/pysatNASA/instruments/cnofs_plp.py
@@ -121,11 +121,9 @@ def clean(self):
 load = cdw.load
 
 # Set the download routine
-basic_tag = {'remote_dir': '/pub/data/cnofs/plp/plasma_1sec/{year:4d}/',
-             'fname': fname}
-download_tags = {'': {'': basic_tag}}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'': {'': 'CNOFS_PLP_PLASMA_1SEC'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/cnofs_vefi.py b/pysatNASA/instruments/cnofs_vefi.py
@@ -124,11 +124,9 @@ def clean(self):
 load = cdw.load
 
 # Set the download routine
-basic_tag = {'remote_dir': '/pub/data/cnofs/vefi/bfield_1sec/{year:4d}/',
-             'fname': fname}
-download_tags = {'': {'dc_b': basic_tag}}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'': {'dc_b': 'CNOFS_VEFI_BFIELD_1SEC'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/de2_lang.py b/pysatNASA/instruments/de2_lang.py
@@ -97,12 +97,9 @@
 load = cdw.load
 
 # Set the download routine
-basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_lang',
-                                    '/plasma500ms_lang_cdaweb/{year:4d}/')),
-             'fname': fname}
-download_tags = {'': {'': basic_tag}}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'': {'': 'DE2_PLASMA500MS_LANG'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/de2_nacs.py b/pysatNASA/instruments/de2_nacs.py
@@ -122,12 +122,9 @@
 load = cdw.load
 
 # Support download routine
-basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/neutral_gas_nacs',
-                                    '/neutral1s_nacs_cdaweb/{year:4d}/')),
-             'fname': fname}
-download_tags = {'': {'': basic_tag}}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'': {'': 'DE2_NEUTRAL1S_NACS'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Support listing files currently on CDAWeb
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py
@@ -109,12 +109,9 @@
 load = cdw.load
 
 # Set the download routine
-basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_rpa',
-                                    '/ion2s_cdaweb/{year:4d}/')),
-             'fname': fname}
-download_tags = {'': {'': basic_tag}}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'': {'': 'DE2_ION2S_RPA'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/de2_wats.py b/pysatNASA/instruments/de2_wats.py
@@ -119,12 +119,9 @@
 load = cdw.load
 
 # Set the download routine
-basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/neutral_gas_wats',
-                                    '/wind2s_wats_cdaweb/{year:4d}/')),
-             'fname': fname}
-download_tags = {'': {'': basic_tag}}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'': {'': 'DE2_WIND2S_WATS'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/formosat1_ivm.py b/pysatNASA/instruments/formosat1_ivm.py
@@ -92,12 +92,9 @@ def init(self):
 load = cdw.load
 
 # Set the download routine
-basic_tag = {'remote_dir': ''.join(('/pub/data/formosat-rocsat/formosat-1',
-                                    '/ipei/{year:4d}/')),
-             'fname': fname}
-download_tags = {'': {'': basic_tag}}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'': {'': 'RS_K0_IPEI'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
diff --git a/pysatNASA/instruments/icon_euv.py b/pysatNASA/instruments/icon_euv.py
@@ -127,13 +127,11 @@ def clean(self):
                                supported_tags=supported_tags)
 
 # Set the download routine
-basic_tag = {'remote_dir': '/pub/data/icon/l2/l2-6_euv/{year:04d}/',
-             'fname': fname}
-download_tags = {'': {'': basic_tag}}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'': {'': 'ICON_L2-6_EUV'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
 
 

diff --git a/pysatNASA/instruments/icon_fuv.py b/pysatNASA/instruments/icon_fuv.py
@@ -119,16 +119,13 @@ def clean(self):
                                supported_tags=supported_tags)
 
 # Set the download routine
-basic_tag24 = {'remote_dir': '/pub/data/icon/l2/l2-4_fuv_day/{year:04d}/',
-               'fname': fname24}
-basic_tag25 = {'remote_dir': '/pub/data/icon/l2/l2-5_fuv_night/{year:04d}/',
-               'fname': fname25}
-download_tags = {'': {'day': basic_tag24, 'night': basic_tag25}}
+download_tags = {'': {'day': 'ICON_L2-4_FUV_DAY',
+                      'night': 'ICON_L2-5_FUV_NIGHT'}}
 
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
 
 

diff --git a/pysatNASA/instruments/icon_ivm.py b/pysatNASA/instruments/icon_ivm.py
@@ -192,13 +192,12 @@ def clean(self):
 
 # Set the download routine
 dirstr = '/pub/data/icon/l2/l2-7_ivm-{id:s}/{{year:4d}}/'
-download_tags = {id: {'': {'remote_dir': dirstr.format(id=id),
-                           'fname': supported_tags[id]['']}}
-                 for id in ['a', 'b']}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'a': {'': 'ICON_L2-7_IVM-A'}, 'b': {'': 'ICON_L2-7_IVM-B'}}
+
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
 
 

diff --git a/pysatNASA/instruments/icon_mighti.py b/pysatNASA/instruments/icon_mighti.py
@@ -226,30 +226,23 @@ def _clean_vars(var_list, flag, min_level):
                                supported_tags=supported_tags)
 
 # Set the download routine
-dirstr1 = '/pub/data/icon/l2/l2-1_mighti-{{id:s}}_los-wind-{color:s}/'
-dirstr2 = '/pub/data/icon/l2/l2-2_mighti_vector-wind-{color:s}/'
-dirstr3 = '/pub/data/icon/l2/l2-3_mighti-{id:s}_temperature/'
-dirnames = {'los_wind_green': dirstr1.format(color='green'),
-            'los_wind_red': dirstr1.format(color='red'),
-            'vector_wind_green': dirstr2.format(color='green'),
-            'vector_wind_red': dirstr2.format(color='red'),
-            'temperature': dirstr3}
-
-download_tags = {}
-for inst_id in supported_tags.keys():
-    download_tags[inst_id] = {}
-    for tag in supported_tags[inst_id].keys():
-        fname = supported_tags[inst_id][tag]
-
-        download_tags[inst_id][tag] = {
-            'remote_dir': ''.join((dirnames[tag].format(id=inst_id),
-                                   '{year:04d}/')),
-            'fname': fname}
-
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'vector':
+                 {'vector_wind_green': 'ICON_L2-2_MIGHTI_VECTOR-WIND-GREEN',
+                  'vector_wind_red': 'ICON_L2-2_MIGHTI_VECTOR-WIND-RED'},
+                 'a':
+                 {'los_wind_green': 'ICON_L2-1_MIGHTI-A_LOS-WIND-GREEN',
+                  'los_wind_red': 'ICON_L2-1_MIGHTI-A_LOS-WIND-RED',
+                  'temperature': 'ICON_L2-3_MIGHTI-A_TEMPERATURE'},
+                 'b':
+                 {'los_wind_green': 'ICON_L2-1_MIGHTI-B_LOS-WIND-GREEN',
+                  'los_wind_red': 'ICON_L2-1_MIGHTI-B_LOS-WIND-RED',
+                  'temperature': 'ICON_L2-3_MIGHTI-B_TEMPERATURE'}}
+
+
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)
 
 

diff --git a/pysatNASA/instruments/iss_fpmu.py b/pysatNASA/instruments/iss_fpmu.py
@@ -115,12 +115,9 @@ def clean(self):
 load = cdw.load
 
 # Set the download routine
-basic_tag = {'remote_dir': ''.join(('/pub/data/international_space_station_iss',
-                                    '/sp_fpmu/{year:4d}/')),
-             'fname': fname}
-download_tags = {'': {'': basic_tag}}
-download = functools.partial(cdw.download, supported_tags=download_tags)
+download_tags = {'': {'': 'ISS_SP_FPMU'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
 
 # Set the list_remote_files routine
-list_remote_files = functools.partial(cdw.list_remote_files,
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
                                       supported_tags=download_tags)