changed test and updated changelog

changelog/4904.bugfix.rst

@@ -1,3 +1,2 @@
-Fix `~.SRSClient` which silently failed to download the SRS files when the tarball for the previous
-years did not exist. Client now actually searches for the tarballs and srs files on the ftp archive
-before returning as results
+Fixed the `~.SRSClient` which silently failed to download the SRS files when the tarball for the previous years did not exist.
+Client now actually searches for the tarballs and srs files on the ftp archive before returning them as results.

sunpy/net/dataretriever/sources/noaa.py

@@ -13,11 +13,10 @@
 from sunpy.net.dataretriever import GenericClient, QueryResponse
 from sunpy.time import TimeRange
 from sunpy.util.parfive_helpers import Downloader
+from sunpy.util.scraper import Scraper
 
 __all__ = ['NOAAIndicesClient', 'NOAAPredictClient', 'SRSClient']
 
-from sunpy.util.scraper import Scraper
-
 
 class NOAAIndicesClient(GenericClient):
     """
@@ -166,7 +165,6 @@ def _get_url_for_timerange(self, timerange):
         Returns a list of urls corresponding to a given time-range.
         """
         result = list()
-
         # Validate time range and return early if out side min/max
         cur_year = Time.now().datetime.year
         req_start_year = timerange.start.datetime.year
@@ -218,19 +216,7 @@ def _get_url_for_timerange(self, timerange):
 
         return result
 
-    # def post_search_hook(self, exdict, matchdict):
-    #     # update the extracted metadata to include the queried times rather
-    #     # than those scraped from the downloaded zip (which includes full year data).
-    #     rowdict = super().post_search_hook(exdict, matchdict)
-    #     rowdict["Start Time"] = matchdict["Start Time"]
-    #     rowdict["End Time"] = matchdict["End Time"]
-    #     rowdict["Start Time"].format = 'iso'
-    #     rowdict["End Time"].format = 'iso'
-    #     return rowdict
-
     def search(self, *args, **kwargs):
-        extractor1 = '{}/warehouse/{:4d}/SRS/{year:4d}{month:2d}{day:2d}SRS.txt'
-        extractor2 = '{}/warehouse/{year:4d}/{}'
         matchdict = self._get_match_dict(*args, **kwargs)
         timerange = TimeRange(matchdict['Start Time'], matchdict['End Time'])
         metalist = []
@@ -253,20 +239,14 @@ def fetch(self, qres, path=None, error_callback=None, **kwargs):
         -------
         Results Object
         """
-
         urls = [qrblock['Url'] for qrblock in qres]
-
         filenames = []
         local_filenames = []
-
-        for i, [url, qre] in enumerate(zip(urls, qres)):
+        for url, qre in zip(urls, qres):
             name = url.split('/')[-1]
-
             day = qre['Start Time']
-
             if name not in filenames:
                 filenames.append(name)
-
             if name.endswith('.gz'):
                 local_filenames.append('{}SRS.txt'.format(day.strftime('%Y%m%d')))
             else:
@@ -284,37 +264,28 @@ def fetch(self, qres, path=None, error_callback=None, **kwargs):
         # OrderedDict is required to maintain ordering because it will be zipped with paths later
         urls = list(OrderedDict.fromkeys(urls))
 
-        dobj = Downloader(max_conn=5)
-
+        downloader = Downloader(max_conn=2)
         for aurl, fname in zip(urls, paths):
-            dobj.enqueue_file(aurl, filename=fname)
+            downloader.enqueue_file(aurl, filename=fname)
 
-        paths = dobj.download()
+        paths = downloader.download()
 
         outfiles = []
         for fname, srs_filename in zip(local_paths, local_filenames):
-
             name = fname.name
-
             past_year = False
-            for i, fname2 in enumerate(paths):
+            for fname2 in paths:
                 fname2 = pathlib.Path(fname2)
-
                 if fname2.name.endswith('.txt'):
                     continue
-
                 year = fname2.name.split('_SRS')[0]
-
                 if year in name:
-                    TarFile = tarfile.open(fname2)
-                    filepath = fname.parent
-                    member = TarFile.getmember('SRS/' + srs_filename)
-                    member.name = name
-                    TarFile.extract(member, path=filepath)
-                    TarFile.close()
-
+                    with tarfile.open(fname2) as open_tar:
+                        filepath = fname.parent
+                        member = open_tar.getmember('SRS/' + srs_filename)
+                        member.name = name
+                        open_tar.extract(member, path=filepath)
                     outfiles.append(fname)
-
                     past_year = True
                     break
 

sunpy/net/dataretriever/sources/tests/test_noaa.py

@@ -137,13 +137,13 @@ def test_fetch(mock_wait, mock_search, mock_enqueue, tmp_path, indices_client):
                                           path / "observed-solar-cycle-indices.json"))
 
 
-@no_vso
 @mock.patch('sunpy.net.dataretriever.sources.noaa.NOAAIndicesClient.search',
             return_value=mock_query_object('2012/10/4', '2012/10/6'))
 # The return value of download is irrelevant
 @mock.patch('parfive.Downloader.download',
             return_value=None)
 @mock.patch('parfive.Downloader.enqueue_file')
+@no_vso
 def test_fido(mock_wait, mock_search, mock_enqueue, tmp_path, indices_client):
     path = tmp_path / "sub"
     path.mkdir()
@@ -159,6 +159,7 @@ def test_fido(mock_wait, mock_search, mock_enqueue, tmp_path, indices_client):
                                           path / "observed-solar-cycle-indices.json"))
 
 
+@no_vso
 @pytest.mark.remote_data
 def test_srs_tar_unpack():
     qr = Fido.search(a.Instrument("soon") & a.Time("2015/01/01", "2015/01/01T23:59:29"))
@@ -167,6 +168,7 @@ def test_srs_tar_unpack():
     assert res.data[0].endswith("20150101SRS.txt")
 
 
+@no_vso
 @pytest.mark.remote_data
 def test_srs_tar_unpack_midyear():
     qr = Fido.search(a.Instrument("soon") & a.Time("2011/06/07", "2011/06/08T23:59:29"))
@@ -191,12 +193,13 @@ def test_srs_missing_tarball(mock_ftp_nlst):
 @pytest.mark.remote_data
 def test_srs_current_year():
     year = datetime.date.today().year
-    qr = Fido.search(a.Instrument("soon") & a.Time(f"{year}/01/01", f"{year}/01/01T23:59:29"))
+    qr = Fido.search(a.Instrument("soon") & a.Time(f"{year}/02/01", f"{year}/02/01T23:59:29"))
     res = Fido.fetch(qr)
     assert len(res) == 1
-    assert res.data[0].endswith(f"{year}0101SRS.txt")
+    assert res.data[0].endswith(f"{year}0201SRS.txt")
 
 
+@no_vso
 @pytest.mark.remote_data
 def test_srs_save_path(tmpdir):
     qr = Fido.search(a.Instrument.srs_table, a.Time("2016/10/01", "2016/10/02"))