diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index 2d7581e..012de9d 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -9,31 +9,35 @@ composition, plasma temperature, and plasma motion may be determined. The DM directly measures the arrival angle of plasma. Using the reported motion of the satellite the angle is converted into ion motion along -two orthogonal directions, perpendicular to the satellite track. +two orthogonal directions, perpendicular to the satellite track. The IVM is +part of the Special Sensor for Ions, Electrons, and Scintillations (SSIES) +instrument suite on DMSP. Downloads data from the National Science Foundation Madrigal Database. The routine is configured to utilize data files with instrument performance flags generated at the Center for Space Sciences at the University of Texas at Dallas. -Parameters +Properties ---------- -platform : string +platform 'dmsp' -name : string +name 'ivm' -tag : string +tag 'utd', None -sat_id : string +sat_id ['f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18'] Example ------- +:: + import pysat dmsp = pysat.Instrument('dmsp', 'ivm', 'utd', 'f15', clean_level='clean') dmsp.download(dt.datetime(2017, 12, 30), dt.datetime(2017, 12, 31), user='Firstname+Lastname', password='email@address.com') - dmsp.load(2017,363) + dmsp.load(2017, 363) Note ---- @@ -41,20 +45,28 @@ Code development supported by NSF grant 1259508 -""" +Custom Functions +---------------- +add_drift_unit_vectors + Add unit vectors for the satellite velocity +add_drifts_polar_cap_x_y + Add polar cap drifts in cartesian coordinates +smooth_ram_drifts + Smooth the ram drifts using a rolling mean +update_DMSP_ephemeris + Updates DMSP instrument data with DMSP ephemeris -from __future__ import print_function -from __future__ import absolute_import +""" import datetime as dt import functools +import logging import numpy as np import pandas as pds from pysatMadrigal.instruments.methods import madrigal as mad_meth from pysat.instruments.methods import general as mm_gen -import logging logger = logging.getLogger(__name__) platform = 'dmsp' @@ -110,15 +122,16 @@ def init(self): self : pysat.Instrument This object - Returns - -------- - Void : (NoneType) - Object modified in place. - - """ logger.info(mad_meth.cedar_rules()) + self.meta.acknowledgements = mad_meth.cedar_rules() + self.meta.references = ' '.join(('F. J. Rich, Users Guide for the Topside', + 'Ionospheric Plasma Monitor (SSIES,', + 'SSIES-2 and SSIES-3) on Spacecraft of', + 'the Defense Meteorological Satellite', + 'Program (Air Force Phillips Laboratory,', + 'Hanscom AFB, MA, 1994), Vol. 1, p. 25.')) return @@ -131,26 +144,20 @@ def download(date_array, tag='', sat_id='', data_path=None, user=None, date_array : array-like list of datetimes to download data for. The sequence of dates need not be contiguous. - tag : string ('') + tag : string Tag identifier used for particular dataset. This input is provided by - pysat. - sat_id : string ('') + pysat. (default='') + sat_id : string Satellite ID string identifier used for particular dataset. This input - is provided by pysat. - data_path : string (None) - Path to directory to download data to. - user : string (None) + is provided by pysat. (default='') + data_path : string + Path to directory to download data to. (default=None) + user : string User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. - - Returns - -------- - Void : (NoneType) - Downloads data to disk. + pysat. If an account is required for dowloads this routine here must + error if user not supplied. (default=None) + password : string + Password for data download. (default=None) Notes ----- @@ -185,15 +192,10 @@ def clean(inst): Parameters ----------- - inst : (pysat.Instrument) + inst : pysat.Instrument Instrument class object, whose attribute clean_level is used to return the desired level of data selectivity. - Returns - -------- - Void : (NoneType) - data in inst is modified in-place. - Notes -------- Supports 'clean', 'dusty', 'dirty' @@ -329,15 +331,15 @@ def add_drift_unit_vectors(inst): inst.data.loc[inst.index[idx], 'unit_cross_x'] *= -1.0 inst.data.loc[inst.index[idx], 'unit_cross_y'] *= -1.0 - inst['unit_ram_r'] = inst['unit_ram_x'] * np.cos(theta) + \ - inst['unit_ram_y'] * np.sin(theta) - inst['unit_ram_theta'] = -inst['unit_ram_x'] * np.sin(theta) + \ - inst['unit_ram_y'] * np.cos(theta) + inst['unit_ram_r'] = (inst['unit_ram_x'] * np.cos(theta) + + inst['unit_ram_y'] * np.sin(theta)) + inst['unit_ram_theta'] = (-inst['unit_ram_x'] * np.sin(theta) + + inst['unit_ram_y'] * np.cos(theta)) - inst['unit_cross_r'] = inst['unit_cross_x'] * np.cos(theta) + \ - inst['unit_cross_y'] * np.sin(theta) - inst['unit_cross_theta'] = -inst['unit_cross_x'] * np.sin(theta) + \ - inst['unit_cross_y'] * np.cos(theta) + inst['unit_cross_r'] = (inst['unit_cross_x'] * np.cos(theta) + + inst['unit_cross_y'] * np.sin(theta)) + inst['unit_cross_theta'] = (-inst['unit_cross_x'] * np.sin(theta) + + inst['unit_cross_y'] * np.cos(theta)) return @@ -365,6 +367,7 @@ def add_drifts_polar_cap_x_y(inst, rpa_flag_key=None, ------- Polar cap drifts assume there is no vertical component to the X-Y velocities + """ # Get the good RPA data, if available @@ -382,10 +385,10 @@ def add_drifts_polar_cap_x_y(inst, rpa_flag_key=None, add_drift_unit_vectors(inst) # Calculate the velocities - inst['ion_vel_pc_x'] = iv_x * inst['unit_ram_x'] + \ - inst[cross_vel_key] * inst['unit_cross_x'] - inst['ion_vel_pc_y'] = iv_x * inst['unit_ram_y'] + \ - inst[cross_vel_key] * inst['unit_cross_y'] + inst['ion_vel_pc_x'] = (iv_x * inst['unit_ram_x'] + + inst[cross_vel_key] * inst['unit_cross_x']) + inst['ion_vel_pc_y'] = (iv_x * inst['unit_ram_y'] + + inst[cross_vel_key] * inst['unit_cross_y']) # Flag the velocities as full (False) or partial (True) inst['partial'] = False diff --git a/pysatMadrigal/instruments/jro_isr.py b/pysatMadrigal/instruments/jro_isr.py index b92d13a..639fb38 100644 --- a/pysatMadrigal/instruments/jro_isr.py +++ b/pysatMadrigal/instruments/jro_isr.py @@ -8,22 +8,25 @@ Downloads data from the JRO Madrigal Database. -Parameters +Properties ---------- -platform : string +platform 'jro' -name : string +name 'isr' -tag : string +tag 'drifts', 'drifts_ave', 'oblique_stan', 'oblique_rand', 'oblique_long' -Example -------- +Examples +-------- +:: + import pysat - dmsp = pysat.Instrument('jro', 'isr', 'drifts', clean_level='clean') - dmsp.download(dt.datetime(2017, 12, 30), dt.datetime(2017, 12, 31), - user='Firstname+Lastname', password='email@address.com') - dmsp.load(2017,363) + jro = pysat.Instrument('jro', 'isr', 'drifts', clean_level='clean') + jro.download(pysat.datetime(2017, 12, 30), pysat.datetime(2017, 12, 31), + user='Firstname+Lastname', password='email@address.com') + jro.load(2017, 363) + Note ---- @@ -104,19 +107,18 @@ def init(self): self : pysat.Instrument This object - Returns - -------- - Void : (NoneType) - Object modified in place. + """ + ackn_str = ' '.join(["The Jicamarca Radio Observatory is a facility of", + "the Instituto Geofisico del Peru operated", + "with support from the NSF AGS-1433968", + "through Cornell University.\n", + mad_meth.cedar_rules()]) - """ + logger.info(ackn_str) + self.meta.acknowledgements = ackn_str + self.meta.references = "?" - logger.info(' '.join(["The Jicamarca Radio Observatory is operated by", - "the Instituto Geofisico del Peru, Ministry of", - "Education, with support from the National Science", - "Foundation as contracted through Cornell", - "University. ", mad_meth.cedar_rules()])) return @@ -129,26 +131,20 @@ def download(date_array, tag='', sat_id='', data_path=None, user=None, date_array : array-like list of datetimes to download data for. The sequence of dates need not be contiguous. - tag : string ('') + tag : string Tag identifier used for particular dataset. This input is provided by - pysat. - sat_id : string ('') + pysat. (default='') + sat_id : string Satellite ID string identifier used for particular dataset. This input - is provided by pysat. - data_path : string (None) - Path to directory to download data to. - user : string (None) + is provided by pysat. (default='') + data_path : string + Path to directory to download data to. (default=None) + user : string User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. - - Returns - -------- - Void : (NoneType) - Downloads data to disk. + pysat. If an account is required for dowloads this routine here must + error if user not supplied. (default=None) + password : string + Password for data download. (default=None) Notes ----- @@ -170,11 +166,6 @@ def download(date_array, tag='', sat_id='', data_path=None, user=None, def clean(self): """Routine to return JRO ISR data cleaned to the specified level - Returns - -------- - Void : (NoneType) - data in inst is modified in-place. - Notes -------- Supports 'clean', 'dusty', 'dirty' diff --git a/pysatMadrigal/instruments/methods/madrigal.py b/pysatMadrigal/instruments/methods/madrigal.py index f812d75..cbd6317 100644 --- a/pysatMadrigal/instruments/methods/madrigal.py +++ b/pysatMadrigal/instruments/methods/madrigal.py @@ -48,14 +48,14 @@ def load(fnames, tag=None, sat_id=None, xarray_coords=[]): fnames : array-like iterable of filename strings, full path, to data files to be loaded. This input is nominally provided by pysat itself. - tag : string ('') + tag : string tag name used to identify particular data set to be loaded. This input is nominally provided by pysat itself. While tag defaults to None here, pysat provides '' as the default - tag unless specified by user at Instrument instantiation. - sat_id : string ('') + tag unless specified by user at Instrument instantiation. (default='') + sat_id : string Satellite ID used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. + This input is nominally provided by pysat itself. (default='') xarray_coords : list List of keywords to use as coordinates if xarray output is desired instead of a Pandas DataFrame (default=[]) @@ -71,8 +71,9 @@ def load(fnames, tag=None, sat_id=None, xarray_coords=[]): Examples -------- :: + inst = pysat.Instrument('jro', 'isr', 'drifts') - inst.load(2010,18) + inst.load(2010, 18) """ @@ -172,34 +173,29 @@ def download(date_array, inst_code=None, kindat=None, data_path=None, date_array : array-like list of datetimes to download data for. The sequence of dates need not be contiguous. - inst_code : string (None) + inst_code : string Madrigal instrument code(s), cast as a string. If multiple are used, - separate them with commas. - kindat : string (None) + separate them with commas. (default=None) + kindat : string Experiment instrument code(s), cast as a string. If multiple are used, - separate them with commas. - data_path : string (None) - Path to directory to download data to. - user : string (None) + separate them with commas. (default=None) + data_path : string + Path to directory to download data to. (default=None) + user : string User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. - url : string ('http://cedar.openmadrigal.org') - URL for Madrigal site - file_format : string ('hdf5') + pysat. If an account is required for dowloads this routine here must + error if user not supplied. (default=None) + password : string + Password for data download. (default=None) + url : string + URL for Madrigal site (default='http://cedar.openmadrigal.org') + file_format : string File format for Madrigal data. Load routines currently only accept 'hdf5', but any of the Madrigal options may be used here. + (default='hdf5') - Returns - -------- - Void : (NoneType) - Downloads data to disk. - - Notes - ----- + Note + ---- The user's names should be provided in field user. Ruby Payne-Scott should be entered as Ruby+Payne-Scott @@ -258,40 +254,37 @@ def get_remote_filenames(inst_code=None, kindat=None, user=None, Parameters ---------- - inst_code : string (None) + inst_code : string Madrigal instrument code(s), cast as a string. If multiple are used, - separate them with commas. - kindat : string (None) + separate them with commas. (default=None) + kindat : string Madrigal experiment code(s), cast as a string. If multiple are used, separate them with commas. If not supplied, all will be returned. - data_path : string (None) - Path to directory to download data to. - user : string (None) + (default=None) + data_path : string + Path to directory to download data to. (default=None) + user : string User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. - web_data : MadrigalData (None) + pysat. If an account is required for dowloads this routine here must + error if user not supplied. (default=None) + password : string + Password for data download. (default=None) + web_data : MadrigalData Open connection to Madrigal database or None (will initiate using url) - url : string ('http://cedar.openmadrigal.org') - URL for Madrigal site + (default=None) + url : string + URL for Madrigal site (default='http://cedar.openmadrigal.org') start : dt.datetime Starting time for file list (defaults to 01-01-1900) stop : dt.datetime Ending time for the file list (defaults to time of run) - date_array : dt.datetime (None) + date_array : dt.datetime Array of datetimes to download data for. The sequence of dates need not be contiguous and will be used instead of start and stop if supplied. + (default=None) - Returns - ------- - Void : (NoneType) - Downloads data to disk. - - Notes - ----- + Note + ---- The user's names should be provided in field user. Ruby Payne-Scott should be entered as Ruby+Payne-Scott @@ -408,46 +401,41 @@ def list_remote_files(tag, sat_id, inst_code=None, kindat=None, user=None, Parameters ---------- - tag : (string or NoneType) + tag : string or NoneType Denotes type of file to load. Accepted types are . (default=None) - sat_id : (string or NoneType) + sat_id : string or NoneType Specifies the satellite ID for a constellation. Not used. (default=None) - inst_code : string (None) + inst_code : string Madrigal instrument code(s), cast as a string. If multiple are used, - separate them with commas. - kindat : string (None) + separate them with commas. (default=None) + kindat : string Madrigal experiment code(s), cast as a string. If multiple are used, separate them with commas. If not supplied, all will be returned. - data_path : string (None) - Path to directory to download data to. - user : string (None) + (default=None) + data_path : string + Path to directory to download data to. (default=None) + user : string User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. - supported_tags : (dict or NoneType) + pysat. If an account is required for dowloads this routine here must + error if user not supplied. (default=None) + password : string + Password for data download. (default=None) + supported_tags : dict or NoneType keys are sat_id, each containing a dict keyed by tag where the values file format template strings. (default=None) - url : string ('http://cedar.openmadrigal.org') - URL for Madrigal site + url : string + URL for Madrigal site (default='http://cedar.openmadrigal.org') two_digit_year_break : int If filenames only store two digits for the year, then '1900' will be added for years >= two_digit_year_break and '2000' will be added for years < two_digit_year_break. - start : (dt.datetime) + start : dt.datetime Starting time for file list (defaults to 01-01-1900) - stop : (dt.datetime) + stop : dt.datetime Ending time for the file list (defaults to time of run) - Returns - ------- - Void : (NoneType) - Downloads data to disk. - Notes ----- The user's names should be provided in field user. Ruby Payne-Scott should @@ -464,6 +452,7 @@ def list_remote_files(tag, sat_id, inst_code=None, kindat=None, user=None, This method is intended to be set in an instrument support file at the top level using functools.partial :: + list_remote_files = functools.partial(mad_meth.list_remote_files, supported_tags=supported_tags, inst_code=madrigal_inst_code) @@ -523,8 +512,11 @@ def filter_data_single_date(self): This routine is intended to be added to the Instrument nanokernel processing queue via + :: + inst = pysat.Instrument() inst.custom.attach(filter_data_single_date, 'modify') + This function will then be automatically applied to the Instrument object data on every load by the pysat nanokernel. @@ -535,7 +527,10 @@ def filter_data_single_date(self): pysat instrument file to this one. within platform_name.py set + :: + default = pysat.instruments.methods.madrigal.filter_data_single_date + at the top level """ diff --git a/pysatMadrigal/instruments/templates/madrigal_pandas.py b/pysatMadrigal/instruments/templates/madrigal_pandas.py index 7fb8141..cca211f 100644 --- a/pysatMadrigal/instruments/templates/madrigal_pandas.py +++ b/pysatMadrigal/instruments/templates/madrigal_pandas.py @@ -20,18 +20,19 @@ Files can be safely downloaded without knowing the file_format keyword, or equivalently, how Madrigal names the files. See `Examples` for more. -Parameters +Properties ---------- -platform : string +platform 'madrigal' -name : string +name 'pandas' -tag : string +tag '' Examples -------- :: + # for isolated use of a madrigal data set import pysat # download DMSP data from Madrigal @@ -40,7 +41,7 @@ madrigal_tag=10241) dmsp.download(dt.datetime(2017, 12, 30), dt.datetime(2017, 12, 31), user='Firstname+Lastname', password='email@address.com') - dmsp.load(2017,363) + dmsp.load(2017, 363) # for users that plan on using multiple Madrigal datasets # using this general interface then an additional parameter @@ -75,16 +76,13 @@ """ -from __future__ import print_function -from __future__ import absolute_import - import datetime as dt import functools +import logging from pysat.instruments.methods import madrigal as mad_meth from pysat.instruments.methods import nasa_cdaweb as cdw -import logging logger = logging.getLogger(__name__) platform = 'madrigal' @@ -94,27 +92,18 @@ # need to sort out test day setting for unit testing _test_dates = {'': {'': dt.datetime(2010, 1, 19)}} -# support list files routine -# use the default CDAWeb method -######### # need a way to get the filename strings for a particular instrument # I've put in wildcards for now.... ######### jro_fname1 = '*{year:4d}{month:02d}{day:02d}' jro_fname2 = '.{version:03d}.hdf5' -supported_tags = {ss: {'': jro_fname1 + "*" + jro_fname2} +supported_tags = {ss: {'': '*'.join((jro_fname1, jro_fname2))} for ss in sat_ids.keys()} list_files = functools.partial(cdw.list_files, supported_tags=supported_tags) -# let pysat know that data is spread across more than one file -# multi_file_day=True - -# Set to False to specify using xarray (not using pandas) -# Set to True if data will be returned via a pandas DataFrame pandas_format = True -# support load routine load = mad_meth.load # support download routine @@ -133,12 +122,6 @@ def init(self): self : pysat.Instrument This object - Returns - -------- - Void : (NoneType) - Object modified in place. - - """ logger.info(mad_meth.cedar_rules()) @@ -166,30 +149,24 @@ def _general_download(date_array, tag='', sat_id='', data_path=None, user=None, date_array : array-like list of datetimes to download data for. The sequence of dates need not be contiguous. - tag : string ('') + tag : string Tag identifier used for particular dataset. This input is provided by - pysat. - sat_id : string ('') + pysat. (default='') + sat_id : string Satellite ID string identifier used for particular dataset. This input - is provided by pysat. - data_path : string (None) - Path to directory to download data to. - user : string (None) + is provided by pysat. (default='') + data_path : string + Path to directory to download data to. (default=None) + user : string User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. + pysat. If an account is required for dowloads this routine here must + error if user not supplied. (default=None) + password : string + Password for data download. (default=None) inst_code : int - Madrigal integer code used to identify platform + Madrigal integer code used to identify platform (default=None) kindat : int - Madrigal integer code used to identify data set - - Returns - -------- - Void : (NoneType) - Downloads data to disk. + Madrigal integer code used to identify data set (default=None) Notes ----- @@ -203,6 +180,7 @@ def _general_download(date_array, tag='', sat_id='', data_path=None, user=None, downloads. """ + mad_meth.download(date_array, inst_code=inst_code, kindat=kindat, data_path=data_path, user=user, password=password) @@ -210,13 +188,8 @@ def _general_download(date_array, tag='', sat_id='', data_path=None, user=None, def clean(self): """Placeholder routine that would normally return cleaned data - Returns - -------- - Void : (NoneType) - data in inst is modified in-place. - - Notes - -------- + Note + ---- Supports 'clean', 'dusty', 'dirty' in the sense that it prints a message noting there is no cleaning. 'None' is also supported as it signifies no cleaning. @@ -224,6 +197,7 @@ def clean(self): Routine is called by pysat, and not by the end user directly. """ + if self.clean_level in ['clean', 'dusty', 'dirty']: logger.warning('Generalized Madrigal data support has no cleaning.')