Backport PR #6887 on branch 4.0 (Adds support in SunPy for the VSO RE…

…ST/TAP Data Providers) (#6934)

Co-authored-by: ejm4567 <mansky@mindspring.com>

changelog/6887.feature.rst

@@ -0,0 +1 @@
+Able to download files from REST/TAP Data Providers from the VSO.

sunpy/net/dataretriever/sources/lyra.py

@@ -36,12 +36,12 @@ class LYRAClient(GenericClient):
     Source: http://vso.stanford.edu/cgi-bin/search
     Total estimated size: 2.914 Gbyte
     <BLANKLINE>
-           Start Time       ...          Info Required
-                            ...
-    ----------------------- ... --------------------------------
-    2016-01-01 09:41:00.000 ... ...
-    2016-01-01 09:41:00.000 ... ...
-    2016-01-01 09:41:00.000 ... ...
+           Start Time               End Time        Source ... Extent Type   Size
+                                                           ...              Mibyte
+    ----------------------- ----------------------- ------ ... ----------- --------
+    2016-01-01 09:41:00.000 2016-01-01 10:40:00.000 PROBA2 ...         N/A  2328.75
+    2016-01-01 09:41:00.000 2016-01-01 10:40:00.000 PROBA2 ...         N/A 419.0625
+    2016-01-01 09:41:00.000 2016-01-01 10:40:00.000 PROBA2 ...         N/A  30.9375
     <BLANKLINE>
     <BLANKLINE>
     """

sunpy/net/vso/table_response.py

@@ -50,7 +50,7 @@ def iter_sort_response(response):
 
 
 class VSOQueryResponseTable(QueryResponseTable):
-    hide_keys = ['fileid', 'fileurl']
+    hide_keys = ['fileid', 'fileurl', 'Info Required']
     errors = TableAttribute(default=[])
     size_column = 'Size'
 

sunpy/net/vso/tests/test_vso.py

@@ -359,3 +359,35 @@ def test_iris_filename(client):
     search_results = client.search(a.Time("2018-01-02 15:31:55", "2018-01-02 15:31:55"), a.Instrument.iris)
     filename = client.mk_filename(pattern, search_results[0], None, url)
     assert filename.endswith("iris_l2_20180102_153155_3610108077_sji_1330_t000_fits.gz")
+
+
+@pytest.mark.remote_data
+def test_table_noinfo_required(client):
+    res = client.search(a.Time('2017/12/17 00:00:00', '2017/12/17 06:00:00'), a.Instrument('aia'), a.Wavelength(171 * u.angstrom))
+    assert 'Info Required' not in res.keys() and len(res) > 0
+
+
+@pytest.mark.remote_data
+def test_table_has_info_required_swap(client):
+    res = client.search(a.Time('2020/02/15 00:00:00', '2020/02/15 20:00:00'), a.Instrument('swap'), a.Provider('ESA'), a.Source('PROBA2'))
+    assert 'Info Required' in res.keys() and len(res) > 0
+
+
+@pytest.mark.remote_data
+def test_table_has_info_required_lyra(client):
+    res = client.search(a.Time('2020/02/15 00:00:00', '2020/02/17 20:00:00'), a.Instrument('lyra'), a.Provider('ESA'), a.Source('PROBA2'))
+    assert 'Info Required' in res.keys() and len(res) > 0
+
+
+@pytest.mark.remote_data
+def test_fetch_swap(client, tmp_path):
+    res = client.search(a.Time('2020/02/15 00:00:00', '2020/02/15 20:00:00'), a.Instrument('swap'), a.Provider('ESA'), a.Source('PROBA2'))
+    files = client.fetch(res[0:1], path=tmp_path)
+    assert len(files) == 1
+
+
+@pytest.mark.remote_data
+def test_fetch_lyra(client, tmp_path):
+    res = client.search(a.Time('2020/02/15 00:00:00', '2020/02/17 20:00:00'), a.Instrument('lyra'), a.Provider('ESA'), a.Source('PROBA2'))
+    files = client.fetch(res[0:1], path=tmp_path)
+    assert len(files) == 1

sunpy/net/vso/vso.py

@@ -418,7 +418,15 @@ def fetch(self, query_response, path=None, methods=None, site=None,
         return results
 
     def make_getdatarequest(self, response, methods=None, info=None):
-        """ Make datarequest with methods from response. """
+        """
+        Make datarequest with methods from response.
+        """
+        # Pass back the Apache session ID to the VSO if it exists in the response
+        for item in response:
+            info_required = item.get("Info Required", None)
+            if info_required is not None:
+                info['required'] = item['Info Required']
+
         if methods is None:
             methods = self.method_order + ['URL']