Unified Downloader

CHANGELOG.md

@@ -18,7 +18,14 @@ Latest
 * Added database url config setting to be setup by default as a sqlite database in the sunpy working directory
 * Added a few tests for the sunpy.roi module
  * Added a few tests for the sunpy.roi module
+ * Added ability to download level3 data for lyra Light Curve along with corresponding tests.
 
+ *Unified Downloader:
+  *Custom Light Curve clients added for downloading files:(EVE,LYRA,NORH,NOAA,RHESSI)
+  *VSO module compatability
+  *Factory based implementation of UnifiedDownloader
+  *VSO esque query style
+
 0.5.0
 -----
 
@@ -37,8 +44,8 @@ Latest
  * Fix algorithm in sunpy.sun.equation_of_center
  * Added Docstrings to LightCurve methods.
  * Added tests for classes in sunpy.map.sources. Note that some classes (TRACE, RHESSI) were left out because SunPy is not able to read their FITS files.
- * Added functions that implement image coalignment with support for MapCubes.
- * Cleaned up the sunpy namespace, removed .units, /ssw and .sphinx. Also moved .coords .physics.transforms.
+ * Added functions that implement image coalignment with support for MapCubes. 
+ * Cleaned up the sunpy namespace, removed .units, /ssw and .sphinx. Also moved .coords .physics.transforms. 
  * Added contains functionality to TimeRange module
  * Added t='now' to parse_time to privide utcnow datetime.
  * Fixed time dependant functions (.sun) to default to t='now'
@@ -55,7 +62,6 @@ Bug Fixes:
     * Some documentation fixes
     * fix file paths to use os.path.join for platform independance.
 
-
 0.4.0
 -----
 Features:
@@ -118,7 +124,7 @@ Major Changes:
  * sunpy.time.timerange has a split() method that divides up a time range into n equal parts.
  * Added download progress bar
  * pyfits is depricated in favor of Astropy
-
+    
 spectra:
 
  * Plotting has been refactorted to use a consistent interface
@@ -132,14 +138,14 @@ Map:
  * attributes of the map class are now read only, changes have to be made through map.meta
  * new MapMeta class to replace MapHeader, MapMeta is not returned by sunpy.io
  * The groundwork for GenericMap inherting from astropy.NDData has been done, there is now a NDDataStandin class to provide basic functionality.
-
-io:
+    
+io:  
  * top level file_tools improved to be more flexible and support multiple HDUs
  * all functions in sunpy.io now assume mutliple HDUs, even JP2 ones.
  * there is now a way to override the automatic filetype detection
  * Automatic fits file detection improved
  * extract_waveunit added to io.fits for detection of common ways of storing wavelength unit in fits files.
-
+      
 
 Bug fixes or under the hood changes:
 
@@ -170,3 +176,4 @@ Below are the main features that have been added for this release:
     * Resample method
     * Superpixel method
     * The addition of the rotate() method for 2D maps.
+

sunpy/instr/rhessi.py

@@ -78,7 +78,7 @@ def get_obssumm_dbase_file(time_range):
     data_location = 'dbase/'
 
     url_root = data_servers[0] + data_location
-    url = url_root + _time_range.start.strftime("hsi_obssumm_filedb_%Y%m.txt")
+    url = url_root + _time_range.start().strftime("hsi_obssumm_filedb_%Y%m.txt")
 
     f = urllib.urlretrieve(url)
 
@@ -181,7 +181,7 @@ def get_obssum_filename(time_range):
     result = parse_obssumm_dbase_file(f[0])
     _time_range = TimeRange(time_range)
 
-    index_number = _time_range.start.day - 1
+    index_number = _time_range.start().day - 1
 
     return data_servers[0] + data_location + result.get('filename')[index_number] + 's'
 

sunpy/lightcurve/sources/eve.py

@@ -57,21 +57,6 @@ def peek(self, column = None, **kwargs):
         figure.show()
         return figure
 
-    @staticmethod
-    def _get_default_uri():
-        """Load latest level 0CS if no other data is specified"""
-        return "http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/LATEST_EVE_L0CS_DIODES_1m.txt"
-
-    @staticmethod
-    def _get_url_for_date(date):
-        """Returns a URL to the EVE data for the specified date
-
-            @NOTE: currently only supports downloading level 0 data
-            .TODO: No data available prior to 2010/03/01!
-        """
-        base_url = 'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/'
-        return base_url + date.strftime('%Y/%Y%m%d') + '_EVE_L0CS_DIODES_1m.txt'
-
     @classmethod
     def _parse_csv(cls, filepath):
         """Parses an EVE CSV file"""
@@ -147,3 +132,4 @@ def _parse_level_0cs(fp):
 
         #data.columns = fields
         return meta, data
+

sunpy/lightcurve/sources/goes.py

@@ -71,85 +71,6 @@ def peek(self, title="GOES Xray Flux"):
 
         return figure
 
-    @classmethod
-    def _get_default_uri(cls):
-        """Retrieve latest GOES data if no other data is specified"""
-        today = datetime.datetime.today()
-        days_back = 3
-        time_range = TimeRange(today - datetime.timedelta(days=days_back),
-                               today - datetime.timedelta(days=days_back - 1))
-        return cls._get_url_for_date_range(time_range)
-
-    @classmethod
-    def _get_goes_sat_num(self, start, end):
-        """Parses the query time to determine which GOES satellite to use."""
-
-        goes_operational = {
-        2: TimeRange('1981-01-01', '1983-04-30'),
-        5: TimeRange('1983-05-02', '1984-07-31'),
-        6: TimeRange('1983-06-01', '1994-08-18'),
-        7: TimeRange('1994-01-01', '1996-08-13'),
-        8: TimeRange('1996-03-21', '2003-06-18'),
-        9: TimeRange('1997-01-01', '1998-09-08'),
-        10: TimeRange('1998-07-10', '2009-12-01'),
-        11: TimeRange('2006-06-20', '2008-02-15'),
-        12: TimeRange('2002-12-13', '2007-05-08'),
-        13: TimeRange('2006-08-01', '2006-08-01'),
-        14: TimeRange('2009-12-02', '2010-10-04'),
-        15: TimeRange('2010-09-01', datetime.datetime.utcnow())}
-
-        sat_list = []
-        for sat_num in goes_operational:
-            if ((start > goes_operational[sat_num].start and
-                 start < goes_operational[sat_num].end and
-                (end > goes_operational[sat_num].start and
-                 end < goes_operational[sat_num].end))):
-                # if true then the satellite with sat_num is available
-                sat_list.append(sat_num)
-
-        if not sat_list:
-            # if no satellites were found then raise an exception
-            raise Exception('No operational GOES satellites within time range')
-        else:
-            return sat_list
-
-    @staticmethod
-    def _get_url_for_date_range(*args):
-        """Returns a URL to the GOES data for the specified date.
-
-        Parameters
-        ----------
-        args : TimeRange, datetimes, date strings
-            Date range should be specified using a TimeRange, or start
-            and end dates at datetime instances or date strings.
-        satellite_number : int
-            GOES satellite number (default = 15)
-        data_type : string
-            Data type to return for the particular GOES satellite. Supported
-            types depend on the satellite number specified. (default = xrs_2s)
-        """
-        # TimeRange
-        if len(args) == 1 and isinstance(args[0], TimeRange):
-            start = args[0].start
-            end = args[0].end
-        elif len(args) == 2:
-            start = parse_time(args[0])
-            end = parse_time(args[1])
-        if end < start:
-            raise ValueError('start time > end time')
-
-        # find out which satellite and datatype to query from the query times
-        sat_num = GOESLightCurve._get_goes_sat_num(start, end)
-        base_url = 'http://umbra.nascom.nasa.gov/goes/fits/'
-
-        if start < parse_time('1999/01/15'):
-            url = base_url + "{date:%Y}/go{sat:02d}{date:%y%m%d}.fits".format(
-                date=start, sat=sat_num[0])
-        else:
-            url = base_url + "{date:%Y}/go{sat:02d}{date:%Y%m%d}.fits".format(
-                date=start, sat=sat_num[0])
-        return url
-
     @staticmethod
     def _parse_fits(filepath):
         """Parses a GOES FITS file from

sunpy/lightcurve/sources/lyra.py

@@ -14,9 +14,6 @@
 from sunpy.time import parse_time
 from sunpy.util.odict import OrderedDict
 
-from sunpy import config
-TIME_FORMAT = config.get("general", "time_format")
-
 __all__ = ['LYRALightCurve']
 
 class LYRALightCurve(LightCurve):
@@ -80,27 +77,7 @@ def peek(self, names=3, **kwargs):
         figure.show()
 
         return figure
-
-
-    @staticmethod
-    def _get_url_for_date(date,**kwargs):
-        """Returns a URL to the LYRA data for the specified date
-        """
-        dt = parse_time(date or datetime.datetime.utcnow())
-
-        # Filename
-        filename = "lyra_{0:%Y%m%d-}000000_lev{1:d}_std.fits".format(
-            dt, kwargs.get('level',2))
-        # URL
-        base_url = "http://proba2.oma.be/lyra/data/bsd/"
-        url_path = urlparse.urljoin(dt.strftime('%Y/%m/%d/'), filename)
-        return urlparse.urljoin(base_url, url_path)
-
-    @classmethod
-    def _get_default_uri(cls):
-        """Look for and download today's LYRA data"""
-        return cls._get_url_for_date(datetime.datetime.utcnow())
-
+
     @staticmethod
     def _parse_fits(filepath):
         """Loads LYRA data from a FITS file"""
@@ -121,17 +98,8 @@ def _parse_fits(filepath):
         start = parse_time(start_str)
         #end = datetime.datetime.strptime(end_str, '%Y-%m-%dT%H:%M:%S.%f')
 
-        # First column are times.  For level 2 data, the units are [s].
-        # For level 3 data, the units are [min]
-        if hdulist[1].header['TUNIT1'] == 's':
-            times = [start + datetime.timedelta(seconds=int(n))
-                     for n in fits_record.field(0)]
-        elif hdulist[1].header['TUNIT1'] == 'MIN':
-            times = [start + datetime.timedelta(minutes=int(n))
-                     for n in fits_record.field(0)]
-        else:
-            raise ValueError("Time unit in LYRA fits file not recognised.  "
-                             "Value = {0}".format(hdulist[1].header['TUNIT1']))
+        # First column are times
+        times = [start + datetime.timedelta(0, n) for n in fits_record.field(0)]
 
         # Rest of columns are the data
         table = {}

sunpy/lightcurve/sources/noaa.py

@@ -75,16 +75,6 @@ def peek(self, axes=None, type='sunspot SWO', **plot_args):
         figure.show()
         return figure
 
-    @classmethod
-    def _get_default_uri(cls):
-        """Return the url to download indices"""
-        return "http://www.swpc.noaa.gov/ftpdir/weekly/RecentIndices.txt"
-
-    @staticmethod
-    def _get_url_for_date_range(*args, **kwargs):
-        """Returns a URL for the specified date."""
-        return NOAAIndicesLightCurve._get_default_uri()
-
     @staticmethod
     def _parse_csv(filepath):
         """Parses an NOAA indices csv"""
@@ -151,16 +141,6 @@ def peek(self, axes=None, **plot_args):
         figure.show()
         return figure
 
-    @classmethod
-    def _get_default_uri(cls):
-        """Return the url to download indices"""
-        return "http://www.swpc.noaa.gov/ftpdir/weekly/Predict.txt"
-
-    @staticmethod
-    def _get_url_for_date_range(*args, **kwargs):
-        """Returns a URL for the specified date."""
-        return NOAAPredictIndicesLightCurve._get_default_uri()
-
     @staticmethod
     def _parse_csv(filepath):
         """Parses an NOAA indices csv"""

sunpy/lightcurve/sources/norh.py

@@ -14,8 +14,6 @@
 from sunpy.time import parse_time
 from sunpy.util.odict import OrderedDict
 
-from sunpy import config
-TIME_FORMAT = config.get("general", "time_format")
 
 __all__ = ['NoRHLightCurve']
 
@@ -46,25 +44,11 @@ def peek(self, **kwargs):
         axes.set_yscale("log")
         axes.set_ylim(1e-4,1)
         axes.set_title('Nobeyama Radioheliograph')
-        axes.set_xlabel('Start time: ' + self.data.index[0].strftime(TIME_FORMAT))
+        axes.set_xlabel('Start time: ' + self.data.index[0].strftime('%Y-%m-%d %H:%M:%S UT'))
         axes.set_ylabel('Correlation')
         axes.legend()
         plt.show()
 
-    @classmethod
-    def _get_url_for_date(cls,date, **kwargs):
-        """This method retrieves the url for NoRH correlation data for the given date."""
-        #default urllib password anonymous@ is not accepted by the NoRH FTP server.
-        #include an accepted password in base url
-        baseurl='ftp://anonymous:mozilla@example.com@solar-pub.nao.ac.jp/pub/nsro/norh/data/tcx/'
-        #date is a datetime object
-        if 'wavelength' in kwargs:
-            if kwargs['wavelength'] == '34':
-                final_url=urlparse.urljoin(baseurl,date.strftime('%Y/%m/tcz%y%m%d'))
-        else:
-            final_url=urlparse.urljoin(baseurl, date.strftime('%Y/%m/tca%y%m%d'))
-
-        return final_url
 
     @staticmethod
     def _parse_fits(filepath):
@@ -85,3 +69,4 @@ def _parse_fits(filepath):
             norh_time.append(obs_start_time + datetime.timedelta(0,s))
 
         return header, pandas.DataFrame(data, index=norh_time)
+

sunpy/lightcurve/sources/rhessi.py

@@ -62,32 +62,6 @@ def peek(self, title="RHESSI Observing Summary Count Rate", **kwargs):
         figure.autofmt_xdate()
         figure.show()
 
-    @classmethod
-    def _get_default_uri(cls):
-        """Retrieve the latest RHESSI data."""
-        today = datetime.datetime.today()
-        days_back = 3
-        time_range = TimeRange(today - datetime.timedelta(days=days_back),
-                               today - datetime.timedelta(days=days_back - 1))
-        return cls._get_url_for_date_range(time_range)
-
-    @staticmethod
-    def _get_url_for_date_range(*args, **kwargs):
-        """Returns a URL to the RHESSI data for the specified date range.
-
-        Parameters
-        ----------
-        args : TimeRange, datetimes, date strings
-            Date range should be specified using a TimeRange, or start
-            and end dates at datetime instances or date strings.
-        """
-        if len(args) == 1 and isinstance(args[0], TimeRange):
-            time_range = args[0]
-        elif len(args) == 2:
-            time_range = TimeRange(parse_time(args[0]), parse_time(args[1]))
-        url = rhessi.get_obssum_filename(time_range)
-        return url
-
     @staticmethod
     def _parse_fits(filepath):
         """Parses a RHESSI FITS file"""

sunpy/net/attrs.py

@@ -0,0 +1,3 @@
+from .vso.attrs import Time,Instrument
+from .vso import attrs as vso
+

sunpy/net/tests/test_download.py

@@ -82,6 +82,7 @@ def _fun(handler):
     path_fun = partial(default_name, tmp)
 
     dw = Downloader(1, 1)
+    _stop = lambda _: dw.stop()
 
     timeout = CalledProxy(dw.stop)
     timer = threading.Timer(60, timeout)