Merge pull request #1354 from Cadair/0.6_prep_online_tests

JSOC notify fixes for 0.6

CHANGELOG.md

@@ -20,6 +20,7 @@ Latest
  * Added a few tests for the sunpy.roi module
  * Refactored mapcube co-alignment functionality.
  * Removed sample data from distribution and added ability to download sample files
+ * Require JSOC request data calls have an email address attached.
  * Calculation of the solar rotation of a point on the Sun as seen from Earth, and its application to the de-rotation of mapcubes.
  * Downloaded files now keep file extensions rather than replacing all periods with underscores
 

sunpy/net/jsoc/__init__.py

@@ -1,4 +1,2 @@
-from __future__ import absolute_import
-
-from .jsoc import *
-from .attrs import *
+from sunpy.net.jsoc.jsoc import *
+from sunpy.net.jsoc.attrs import *

sunpy/net/jsoc/attrs.py

@@ -1,13 +1,12 @@
 from __future__ import absolute_import
 
-import numpy as np
 import astropy.units as u
 
 from sunpy.net.attr import (Attr, AttrWalker, AttrAnd, AttrOr)
 from sunpy.net.vso.attrs import Time, _VSOSimpleAttr
 
 __all__ = ['Series', 'Protocol', 'Notify', 'Compression', 'Wavelength', 'Time',
-           'Segment', 'walker']
+           'Segment']
 
 
 class Time(Time):
@@ -47,7 +46,11 @@ class Notify(_VSOSimpleAttr):
     """
     An email address to get a notification to when JSOC has staged your request
     """
-    pass
+    def __init__(self, value):
+        super(Notify, self).__init__(value)
+        if value.find('@') == -1:
+            raise ValueError("Notify attribute must contain an '@' symbol to be a valid email address")
+        self.value = value
 
 
 class Compression(_VSOSimpleAttr):
@@ -99,20 +102,20 @@ def _apply(wlk, query, imap):
 
 
 @walker.add_applier(_VSOSimpleAttr)
-def _apply(wlk, query, imap):
+def _apply1(wlk, query, imap):
 
     imap[query.__class__.__name__.lower()] = query.value
 
 
 @walker.add_applier(Time)
-def _apply(wlk, query, imap):
+def _apply2(wlk, query, imap):
 
     imap['start_time'] = query.start
     imap['end_time'] = query.end
 
 
 @walker.add_creator(AttrOr)
-def _create(wlk, query):
+def _create1(wlk, query):
 
     qblocks = []
     for iattr in query.attrs:

sunpy/net/jsoc/jsoc.py

@@ -68,6 +68,10 @@ class JSOCClient(object):
     method, but they can be seperated if you are performing a large or complex
     query.
 
+    .. warning::
+        JSOC now requires you to register your email address before requesting
+        data. See this site: http://jsoc.stanford.edu/ajax/register_email.html
+
     Notes
     -----
     This Client mocks input to this site: http://jsoc.stanford.edu/ajax/exportdata.html
@@ -77,6 +81,7 @@ class JSOCClient(object):
     You can build more complex queries by specifiying parameters to POST to JSOC via keyword
     arguments. You can generate these kwargs using the Export Data page at JSOC.
 
+
     Examples
     --------
 
@@ -86,7 +91,7 @@ class JSOCClient(object):
 
     >>> from sunpy.net import jsoc
     >>> client = jsoc.JSOCClient()
-    >>> response = client.query(jsoc.Time('2010-01-01T00:00', '2010-01-01T01:00'),
+    >>> response = client.query(jsoc.Time('2014-01-01T00:00:00', '2014-01-01T01:00:00'),
     ...                         jsoc.Series('hmi.m_45s'))
 
     the response object holds the records that your query will return:
@@ -114,11 +119,12 @@ class JSOCClient(object):
     Query the JSOC for some AIA 171 data, and seperate out the staging and the
     download steps:
 
+    >>> import astropy.units as u
     >>> from sunpy.net import jsoc
     >>> client = jsoc.JSOCClient()
     >>> response = client.query(jsoc.Time('2014/1/1T00:00:00', '2014/1/1T00:00:36'),
                                 jsoc.Series('aia.lev1_euv_12s'), jsoc.Segment('image'),
-                                jsoc.Wave(171))
+                                jsoc.Wavelength(171*u.AA))
 
     the response object holds the records that your query will return:
 
@@ -238,7 +244,7 @@ def request_data(self, jsoc_response, **kwargs):
 
     def check_request(self, requestIDs):
         """
-        Check the status of a request and print out a messgae about it
+        Check the status of a request and print out a message about it
 
         Parameters
         ----------
@@ -317,8 +323,6 @@ def get(self, jsoc_response, path=None, overwrite=False, progress=True,
         jsoc_response.requestIDs = requestIDs
         time.sleep(sleep/2.)
 
-        r = Results(lambda x: None)
-
         while requestIDs:
             for i, request_id in enumerate(requestIDs):
                 u = self._request_status(request_id)
@@ -329,7 +333,7 @@ def get(self, jsoc_response, path=None, overwrite=False, progress=True,
                 if u.status_code == 200 and u.json()['status'] == '0':
                     rID = requestIDs.pop(i)
                     r = self.get_request(rID, path=path, overwrite=overwrite,
-                                 progress=progress, results=r)
+                                 progress=progress)
 
                 else:
                     time.sleep(sleep)
@@ -386,7 +390,7 @@ def get_request(self, requestIDs, path=None, overwrite=False, progress=True,
         # A Results object tracks the number of downloads requested and the
         # number that have been completed.
         if results is None:
-            results = Results(lambda x: None)
+            results = Results(lambda _: downloader.stop())
 
         urls = []
         for request_id in requestIDs:
@@ -397,6 +401,12 @@ def get_request(self, requestIDs, path=None, overwrite=False, progress=True,
                     if overwrite or not os.path.isfile(os.path.join(path, ar['filename'])):
                         urls.append(urlparse.urljoin(BASE_DL_URL + u.json()['dir'] +
                                                      '/', ar['filename']))
+
+                    else:
+                        print("Skipping download of file {} as it has already been downloaded".format(ar['filename']))
+                        # Add the file on disk to the output
+                        results.map_.update({ar['filename']:{'path':os.path.join(path, ar['filename'])}})
+
                 if progress:
                     print("{0} URLs found for Download. Totalling {1}MB".format(
                                                   len(urls), u.json()['size']))
@@ -406,8 +416,9 @@ def get_request(self, requestIDs, path=None, overwrite=False, progress=True,
                     self.check_request(request_id)
 
         if urls:
-            for url, rcall in list(zip(urls, list(map(lambda x: results.require([x]), urls)))):
-                downloader.download(url, callback=rcall, path=path)
+            for url in urls:
+                downloader.download(url, callback=results.require([url]),
+                                    errback=lambda x: print(x), path=path)
 
         else:
             #Make Results think it has finished.
@@ -463,7 +474,7 @@ def _make_recordset(self, start_time, end_time, series, wavelength='',
 
         return dataset
 
-    def _make_query_payload(self, start_time, end_time, series, notify='',
+    def _make_query_payload(self, start_time, end_time, series, notify=None,
                             protocol='FITS', compression='rice', **kwargs):
         """
         Build the POST payload for the query parameters
@@ -476,6 +487,10 @@ def _make_query_payload(self, start_time, end_time, series, notify='',
         else:
             jprotocol = protocol
 
+        if not notify:
+            raise ValueError("JSOC queries now require a valid email address "
+                             "before they will be accepted by the server")
+
         dataset = self._make_recordset(start_time, end_time, series, **kwargs)
         kwargs.pop('wavelength', None)
 
@@ -493,7 +508,7 @@ def _make_query_payload(self, start_time, end_time, series, notify='',
         payload.update(kwargs)
         return payload
 
-    def _send_jsoc_request(self, start_time, end_time, series, notify='',
+    def _send_jsoc_request(self, start_time, end_time, series, notify=None,
                            protocol='FITS', compression='rice', **kwargs):
         """
         Request that JSOC stages data for download

sunpy/net/jsoc/tests/test_jsoc.py

@@ -5,6 +5,7 @@
 @author: stuart
 """
 import time
+import tempfile
 import datetime
 import astropy.table
 import astropy.time
@@ -34,19 +35,20 @@ def test_jsocresponse_single():
     assert all(j1.table == astropy.table.Table(data=[[1,2,3,4]]))
     assert len(j1) == 4
 
+
 def test_payload():
     start = parse_time('2012/1/1T00:00:00')
     end = parse_time('2012/1/1T00:00:45')
 
-    payload = client._make_query_payload(start, end, 'hmi.M_42s')
+    payload = client._make_query_payload(start, end, 'hmi.M_42s', notify='@')
 
     payload_expected = {
        'ds': '{0}[{1}-{2}]'.format('hmi.M_42s',
                                    start.strftime("%Y.%m.%d_%H:%M:%S_TAI"),
                                    end.strftime("%Y.%m.%d_%H:%M:%S_TAI")),
        'format': 'json',
        'method': 'url',
-       'notify': '',
+       'notify': '@',
        'op': 'exp_request',
        'process': 'n=0|no_op',
        'protocol': 'FITS,compress Rice',
@@ -60,14 +62,15 @@ def test_payload_nocompression():
     start = parse_time('2012/1/1T00:00:00')
     end = parse_time('2012/1/1T00:00:45')
 
-    payload = client._make_query_payload(start, end, 'hmi.M_42s', compression=None)
+    payload = client._make_query_payload(start, end, 'hmi.M_42s',
+                                         compression=None, notify='jsoc@cadair.com')
 
     payload_expected = {
        'ds':'{0}[{1}-{2}]'.format('hmi.M_42s', start.strftime("%Y.%m.%d_%H:%M:%S_TAI"),
                                        end.strftime("%Y.%m.%d_%H:%M:%S_TAI")),
        'format':'json',
        'method':'url',
-       'notify':'',
+       'notify':'jsoc@cadair.com',
        'op':'exp_request',
        'process':'n=0|no_op',
        'protocol':'FITS, **NONE**',
@@ -81,14 +84,15 @@ def test_payload_protocol():
     start = parse_time('2012/1/1T00:00:00')
     end = parse_time('2012/1/1T00:00:45')
 
-    payload = client._make_query_payload(start, end, 'hmi.M_42s', protocol='as-is')
+    payload = client._make_query_payload(start, end, 'hmi.M_42s', protocol='as-is',
+                                         notify='jsoc@cadair.com')
 
     payload_expected = {
        'ds':'{0}[{1}-{2}]'.format('hmi.M_42s', start.strftime("%Y.%m.%d_%H:%M:%S_TAI"),
                                        end.strftime("%Y.%m.%d_%H:%M:%S_TAI")),
        'format':'json',
        'method':'url',
-       'notify':'',
+       'notify':'jsoc@cadair.com',
        'op':'exp_request',
        'process':'n=0|no_op',
        'protocol':'as-is',
@@ -139,7 +143,7 @@ def test_query():
 @pytest.mark.online
 def test_post_pass():
     responses = client.query(attrs.Time('2012/1/1T00:00:00', '2012/1/1T00:00:45'),
-                             attrs.Series('hmi.M_45s'))
+                             attrs.Series('hmi.M_45s'), attrs.Notify('jsoc@cadair.com'))
     aa = client.request_data(responses, return_resp=True)
     tmpresp = aa[0].json()
     assert tmpresp['status'] == 2
@@ -150,7 +154,7 @@ def test_post_pass():
 @pytest.mark.online
 def test_post_wavelength():
     responses = client.query(attrs.Time('2010/07/30T13:30:00','2010/07/30T14:00:00'),attrs.Series('aia.lev1_euv_12s'),
-                             attrs.Wavelength(193*u.AA)|attrs.Wavelength(335*u.AA))
+                             attrs.Wavelength(193*u.AA)|attrs.Wavelength(335*u.AA), attrs.Notify('jsoc@cadair.com'))
     aa = client.request_data(responses, return_resp=True)
     tmpresp = aa[0].json()
     assert tmpresp['status'] == 2
@@ -168,7 +172,8 @@ def test_post_wave_series():
 
 @pytest.mark.online
 def test_post_fail(recwarn):
-    res = client.query(attrs.Time('2012/1/1T00:00:00', '2012/1/1T00:00:45'), attrs.Series('none'))
+    res = client.query(attrs.Time('2012/1/1T00:00:00', '2012/1/1T00:00:45'),
+                       attrs.Series('none'), attrs.Notify('jsoc@cadair.com'))
     client.request_data(res, return_resp=True)
     w = recwarn.pop(Warning)
     assert issubclass(w.category, Warning)
@@ -185,34 +190,36 @@ def test_request_status_fail():
 
 
 @pytest.mark.online
-@pytest.mark.xfail
+#@pytest.mark.xfail
 def test_wait_get():
-    responses = client.query(attrs.Time('2012/1/3T00:00:00', '2012/1/3T00:00:45'), attrs.Series( 'hmi.M_45s'))
-    res = client.get(responses)
+    responses = client.query(attrs.Time('2012/1/1T1:00:36', '2012/1/1T01:00:38'),
+                             attrs.Series( 'hmi.M_45s'), attrs.Notify('jsoc@cadair.com'))
+    path = tempfile.mkdtemp()
+    res = client.get(responses, path=path)
     assert isinstance(res, Results)
-    assert res.total == 2
-
-@pytest.mark.online
-@pytest.mark.xfail
-def test_check_request():
-    responses = client.query(attrs.Time('2012/1/1T01:00:00', '2012/1/1T01:00:45'),
-                             attrs.Series('hmi.M_45s'))
+    assert res.total == 1
 
-    bb = client.request_data(responses)
-    aa = client.check_request(bb)
-    assert aa == [6] or aa == [1] #Incase JSOC is being very efficient
-    time.sleep(5)
-    aa = client.check_request(bb)
-    assert aa == [1]
 
 @pytest.mark.online
 def test_get_request():
-    responses = client.query(attrs.Time('2012/1/1T01:00:00', '2012/1/1T01:00:45'),
-                             attrs.Series('hmi.M_45s'))
+    responses = client.query(attrs.Time('2012/1/1T1:00:36', '2012/1/1T01:00:38'),
+                             attrs.Series('hmi.M_45s'), attrs.Notify('jsoc@cadair.com'))
 
     bb = client.request_data(responses)
-    aa = client.get_request(bb)
+    path = tempfile.mkdtemp()
+    aa = client.get_request(bb, path=path)
+    assert isinstance(aa, Results)
+
+@pytest.mark.online
+@pytest.mark.xfail
+def test_results_filenames():
+    responses = client.query(attrs.Time('2014/1/1T1:00:36', '2014/1/1T01:01:38'),
+                             attrs.Series('hmi.M_45s'), attrs.Notify('jsoc@cadair.com'))
+    path = tempfile.mkdtemp()
+    aa = client.get(responses, path=path)
     assert isinstance(aa, Results)
+    files = aa.wait()
+    assert len(files) == len(responses)
 
 @pytest.mark.online
 def test_invalid_query():