Merge pull request #2364 from Cadair/xrsclient_table

Fix XRSClient time in query results

CHANGELOG.rst

@@ -1,9 +1,14 @@
-0.8.3 (Unreleased)
+0.9.0 (Unreleased)
 ==================
 
+New Features
+------------
+
 Bug Fixes
 ---------
 
+- `~sunpy.net.dataretriever.clients.XRSClient` now reports time ranges of files correctly. [#2364]
+- Make parse_time work with datetime64s and pandas series [#2370]
 - CompositeMap axes scaling now uses map spatial units [#2310]
 - Moved license file to root of repository and updated README file [#2326]
 - Fix docstring formatting for net.vso.attrs [#2309]]
@@ -18,23 +23,8 @@ Bug Fixes
 - Updated masking brightest pixel example [#2338]
 - Changed TRAVIS cronjobs [#2338]
 - Support array values for `obstime` for coordinates and transformations [#2342]
-- Make parse_time work with datetime64s and pandas series [#2370]
-
-0.8.2
-=====
-
-Bug Fixes
----------
-
 - Shows a warning if observation time is missing [#2293]
 - Updates MapCube to access the correct properties of the namedtuple SpatialPair [#2297]
-
-0.8.1
-======
-
-Bug fixes
----------
-
 - Fixed TimeSeries test failures due to missing test files [#2273]
 - Refactored a GOES test to avoid a Py3.6 issue [#2276]
 

sunpy/net/dataretriever/sources/goes.py

@@ -2,12 +2,15 @@
 # This module was developed under funding provided by
 # Google Summer of Code 2014
 
+import os
 import datetime
 
 from sunpy.time import parse_time, TimeRange
 
 from ..client import GenericClient
 
+from sunpy.extern.six.moves.urllib.parse import urlsplit
+
 from sunpy import config
 TIME_FORMAT = config.get("general", "time_format")
 
@@ -54,6 +57,25 @@ def _get_goes_sat_num(self, date):
             raise ValueError('No operational GOES satellites on {}'.format(
                 date.strftime(TIME_FORMAT)))
 
+    def _get_time_for_url(self, urls):
+        times = []
+        for uri in urls:
+            uripath = urlsplit(uri).path
+
+            # Extract the yymmdd or yyyymmdd timestamp
+            datestamp = os.path.splitext(os.path.split(uripath)[1])[0][4:]
+
+            # 1999-01-15 as an integer.
+            if int(datestamp) < 990115:
+                start = datetime.datetime.strptime(datestamp, "%y%m%d")
+            else:
+                start = datetime.datetime.strptime(datestamp, "%Y%m%d")
+
+            almost_day = datetime.timedelta(days=1, milliseconds=-1)
+            times.append(TimeRange(start, start + almost_day))
+
+        return times
+
     def _get_url_for_timerange(self, timerange, **kwargs):
         """
         Returns a URL to the GOES data for the specified date.
@@ -72,7 +94,9 @@ def _get_url_for_timerange(self, timerange, **kwargs):
         base_url = 'http://umbra.nascom.nasa.gov/goes/fits/'
         start_time = datetime.datetime.combine(timerange.start.date(),
                                                datetime.datetime.min.time())
-        total_days = int(timerange.days.value) + 1
+        # make sure we are counting a day even if only a part of it is in the query range.
+        day_range = TimeRange(timerange.start.date(), timerange.end.date())
+        total_days = int(day_range.days.value) + 1
         result = list()
 
         # Iterate over each day in the input timerange and generate a URL for

sunpy/net/dataretriever/tests/test_goes_ud.py

@@ -1,4 +1,5 @@
 import pytest
+import datetime
 
 from sunpy.time.timerange import TimeRange, parse_time
 from sunpy.net.vso.attrs import Time, Instrument
@@ -66,8 +67,13 @@ def test_fixed_satellite():
 def test_query(time):
     qr1 = LCClient.search(time, Instrument('XRS'))
     assert isinstance(qr1, QueryResponse)
-    assert qr1.time_range().start == time.start
-    assert qr1.time_range().end == time.end
+    # We only compare dates here as the start time of the qr will always be the
+    # start of the day.
+    assert qr1.time_range().start.date() == time.start.date()
+
+    almost_day = datetime.timedelta(days=1, milliseconds=-1)
+    end = datetime.datetime.combine(time.end.date(), datetime.time()) + almost_day
+    assert qr1.time_range().end == end
 
 
 def test_query_error():
@@ -109,3 +115,15 @@ def test_fido(time, instrument):
     assert isinstance(qr, UnifiedResponse)
     response = Fido.fetch(qr)
     assert len(response) == qr._numfile
+
+
+@given(goes_time())
+def test_time_for_url(time):
+    time = time.start.date().strftime("%Y/%m/%d")
+    almost_day = datetime.timedelta(days=1, milliseconds=-1)
+
+    tr = TimeRange(time, almost_day)
+    url = LCClient._get_url_for_timerange(tr)
+    times = LCClient._get_time_for_url(url)
+
+    assert all([tr == t2 for t2 in times])

sunpy/net/tests/test_fido.py

@@ -77,8 +77,9 @@ def check_response(query, unifiedresp):
         raise ValueError("No Time Specified")
 
     for block in unifiedresp.responses:
+        res_tr = block.time_range()
         for res in block:
-            assert res.time.start in query_tr
+            assert res.time.start in res_tr
             assert query_instr.lower() == res.instrument.lower()
 
 

sunpy/time/time.py

@@ -1,7 +1,6 @@
 from __future__ import absolute_import, division, print_function
 import re
-from datetime import datetime
-from datetime import timedelta
+from datetime import datetime, date, time, timedelta
 
 import numpy as np
 import pandas
@@ -170,6 +169,8 @@ def parse_time(time_string, time_format='', **kwargs):
         return time_string._mpl_repr()
     elif isinstance(time_string, datetime) or time_format == 'datetime':
         return time_string
+    elif isinstance(time_string, date):
+        return datetime.combine(time_string, time())
     elif isinstance(time_string, tuple):
         return datetime(*time_string)
     elif time_format == 'utime' or isinstance(time_string, (int, float)):