diff --git a/.gitignore b/.gitignore index e990b3810c7..f0525ae06e0 100644 --- a/.gitignore +++ b/.gitignore @@ -281,11 +281,15 @@ package.json # Log files generated by 'vagrant up' *.log + # asv stuff asv_env asv_results html + +# Figure tests results result_images + # Save from Map.save example aia171.fits diff --git a/changelog/7463.feature.rst b/changelog/7463.feature.rst new file mode 100644 index 00000000000..beb9d353b82 --- /dev/null +++ b/changelog/7463.feature.rst @@ -0,0 +1 @@ +A new client (`sunpy.net.dataretriever.ADAPTClient`) has been added to search and download `ADAPT `__ files. diff --git a/docs/reference/net.rst b/docs/reference/net.rst index 833e7db1eb3..bbe5e7c5601 100644 --- a/docs/reference/net.rst +++ b/docs/reference/net.rst @@ -39,6 +39,9 @@ Dataretriever .. automodapi:: sunpy.net.dataretriever.attrs.goes :headings: ^" +.. automodapi:: sunpy.net.dataretriever.attrs.adapt + :headings: ^" + JSOC ---- diff --git a/docs/topic_guide/extending_fido.rst b/docs/topic_guide/extending_fido.rst index 6efda41f478..545647f85f8 100644 --- a/docs/topic_guide/extending_fido.rst +++ b/docs/topic_guide/extending_fido.rst @@ -382,6 +382,7 @@ A simple example, which just checks the type of ``attrs`` and not their values w return supported_attrs.issuperset(query_attrs) Note, that this method is a class method, it gets called without instantiating your client to speed up the dispatching. +If you are using the `~sunpy.net.dataretriever.client.GenericClient` as a base class, you do not need to implement this method, as it is already implemented in the base class. Writing a Fetch Method ---------------------- diff --git a/docs/tutorial/acquiring_data/index.rst b/docs/tutorial/acquiring_data/index.rst index 302c0fb0cbb..0e664797b05 100644 --- a/docs/tutorial/acquiring_data/index.rst +++ b/docs/tutorial/acquiring_data/index.rst @@ -44,9 +44,10 @@ Fido supports a number of different remote data sources. To see a list the Fido For details of using `~sunpy.net.Fido` see :ref:`sunpy-tutorial-acquiring-data-index`. - Client Description - ----------------- ------------------------------------------------------------------------------------------------------- + Client Description + ----------------- ----------------------------------------------------------------------------------------------------------------------- CDAWEBClient Provides access to query and download from the Coordinated Data Analysis Web (CDAWeb). + ADAPTClient Provides access to the ADvanced Adaptive Prediction Technique (ADAPT) products of the National Solar Observatory (NSO). EVEClient Provides access to Level 0CS Extreme ultraviolet Variability Experiment (EVE) data. GBMClient Provides access to data from the Gamma-Ray Burst Monitor (GBM) instrument on board the Fermi satellite. XRSClient Provides access to several GOES XRS files archive. @@ -94,17 +95,17 @@ As an example: Specifies the Instrument name for the search. - Attribute Name Client Full Name Description - --------------------------- ----------- ------------------------ -------------------------------------------------------------------------------- - aia VSO AIA Atmospheric Imaging Assembly - bcs VSO BCS Bragg Crystal Spectrometer - be_continuum VSO BE-Continuum INAF-OACT Barra Equatoriale Continuum Instrument - be_halpha VSO BE-Halpha INAF-OACT Barra Equatoriale Hα Instrument - bigbear VSO Big Bear Big Bear Solar Observatory, California TON and GONG+ sites - caii VSO CAII Kanzelhöhe Ca II k Instrument - cds VSO CDS Coronal Diagnostic Spectrometer - celias VSO CELIAS Charge, Element, and Isotope Analysis System + Attribute Name Client ... Description + --------------------------- ----------- ... -------------------------------------------------------------------------------- + adapt ADAPT ... ADvanced Adaptive Prediction Technique. + aia VSO ... Atmospheric Imaging Assembly + bcs VSO ... Bragg Crystal Spectrometer + be_continuum VSO ... INAF-OACT Barra Equatoriale Continuum Instrument + be_halpha VSO ... INAF-OACT Barra Equatoriale Hα Instrument + bigbear VSO ... Big Bear Solar Observatory, California TON and GONG+ sites ... + xrs XRS ... GOES X-ray Sensor + xrt VSO ... X-Ray Telescope This is a full list of known values, a description, and which clients support those values (if you want to search using a specific data source). Printing attributes like this is supported for most attributes, including client specific ones. diff --git a/docs/whatsnew/6.0.rst b/docs/whatsnew/6.0.rst index dcee02c7697..8c11a23f34a 100644 --- a/docs/whatsnew/6.0.rst +++ b/docs/whatsnew/6.0.rst @@ -88,3 +88,9 @@ Deprecate positional arguments in :meth:`sunpy.map.GenericMap.plot` The arguments for :meth:`sunpy.map.GenericMap.plot` have been changed to being keyword only. Pass them as keyword arguments (e.g., ``..., title=True, ...``) instead. + +Support for ADvanced Adaptive Prediction Technique (ADAPT) +========================================================== + +A new map source has been added (``sunpy.map.sources.ADAPTMap``) to support the ADAPT data files. +In addition, a new client (`sunpy.net.dataretriever.ADAPTClient`) has been added to search and download ADAPT files. diff --git a/sunpy/net/attrs.py b/sunpy/net/attrs.py index c173378dac2..f93d4fab8c8 100644 --- a/sunpy/net/attrs.py +++ b/sunpy/net/attrs.py @@ -15,12 +15,12 @@ In addition to the core attrs defined here, other sunpy clients also provide attrs specific to them, under: -* `a.vso ` -* `a.jsoc ` +* `a.adapt ` * `a.goes ` * `a.hek ` * `a.helio ` - +* `a.jsoc ` +* `a.vso ` """ from ._attrs import ( Detector, diff --git a/sunpy/net/dataretriever/__init__.py b/sunpy/net/dataretriever/__init__.py index 14ec257618e..9ccbb29d49d 100644 --- a/sunpy/net/dataretriever/__init__.py +++ b/sunpy/net/dataretriever/__init__.py @@ -9,6 +9,7 @@ """ from .client import * +from .sources.adapt import * from .sources.eve import * from .sources.fermi_gbm import * from .sources.goes import * diff --git a/sunpy/net/dataretriever/attrs/__init__.py b/sunpy/net/dataretriever/attrs/__init__.py index fe78aa6b36a..42bbf0ee283 100644 --- a/sunpy/net/dataretriever/attrs/__init__.py +++ b/sunpy/net/dataretriever/attrs/__init__.py @@ -1,3 +1,3 @@ -from . import goes +from . import adapt, goes -__all__ = ['goes'] +__all__ = ['adapt', 'goes'] diff --git a/sunpy/net/dataretriever/attrs/adapt.py b/sunpy/net/dataretriever/attrs/adapt.py new file mode 100644 index 00000000000..49b36d5102a --- /dev/null +++ b/sunpy/net/dataretriever/attrs/adapt.py @@ -0,0 +1,77 @@ +from sunpy.net.attr import SimpleAttr + +__all__ = ['ADAPTFileType', 'ADAPTLonType', 'ADAPTInputSource', + 'ADAPTDataAssimilation', 'ADAPTResolution', 'ADAPTVersionYear', 'ADAPTVersionMonth', + 'ADAPTEvolutionMode', 'ADAPTHelioData', 'ADAPTRealizations', 'ADAPTMagData'] + +# Define a custom __dir__ to restrict tab-completion to __all__ +def __dir__(): + return __all__ + + +class ADAPTFileType(SimpleAttr): + """ + ADAPT file type: Public. + """ + pass + +class ADAPTLonType(SimpleAttr): + """ + ADAPT longitude type: Carrington Fixed, Central Meridian, East Limb. + """ + pass + +class ADAPTInputSource(SimpleAttr): + """ + ADAPT input source: All, KPVT, VSM, GONG, HMI, MDI, MWO. + """ + pass + +class ADAPTDataAssimilation(SimpleAttr): + """ + ADAPT data assimilation: WH, enLS, enkf, enLAKF. + """ + pass + +class ADAPTResolution(SimpleAttr): + """ + ADAPT model spatial resolution: 1.0 deg, 0.2 deg. + """ + pass + +class ADAPTVersionYear(SimpleAttr): + """ + ADAPT code version year. + """ + pass + +class ADAPTVersionMonth(SimpleAttr): + """ + ADAPT code version month. + """ + pass + +class ADAPTEvolutionMode(SimpleAttr): + """ + ADAPT evolution mode: Data assimilation step, Intermediate step, Forecast step. + """ + pass + +class ADAPTHelioData(SimpleAttr): + """ + ADAPT helioseismic data: Not added or no data, Far-side, Emergence, Both emergence & far-side. + """ + pass + +class ADAPTRealizations(SimpleAttr): + """ + ADAPT realizations: number of model/file realizations, e.g., 16 -> "016" + """ + pass + + +class ADAPTMagData(SimpleAttr): + """ + ADAPT magnetic data: Not added or no data, Mag-los, Mag-vector, Mag- both los & vector, Mag- polar avg obs, Mag- los & polar, Mag- vector & polar, Mag- both los and vector & polar. + """ + pass diff --git a/sunpy/net/dataretriever/sources/adapt.py b/sunpy/net/dataretriever/sources/adapt.py new file mode 100644 index 00000000000..b5e0036bd8b --- /dev/null +++ b/sunpy/net/dataretriever/sources/adapt.py @@ -0,0 +1,94 @@ +from sunpy.net import attrs as a +from sunpy.net.dataretriever import GenericClient +from sunpy.net.dataretriever.attrs.adapt import ( + ADAPTDataAssimilation, + ADAPTEvolutionMode, + ADAPTFileType, + ADAPTHelioData, + ADAPTInputSource, + ADAPTLonType, + ADAPTMagData, + ADAPTRealizations, + ADAPTResolution, + ADAPTVersionMonth, + ADAPTVersionYear, +) + +__all__ = ['ADAPTClient'] + + +class ADAPTClient(GenericClient): + """ + Provides access to the ADvanced Adaptive Prediction Technique (ADAPT) products + of the National Solar Observatory (NSO). + + `Searches data hosted by the NSO `__ + + Examples + -------- + >>> import astropy.units as u + + >>> from sunpy.net import Fido, attrs as a + >>> from sunpy.coordinates.sun import carrington_rotation_time + + >>> # Define the Carrington Rotation Number and the number of frames + >>> CR = 2193 + >>> frames = 10 + >>> date_start = carrington_rotation_time(CR) + >>> date_end = date_start + frames*(3*1.9999999 * u.hour) + >>> longitude_type = '0' + + >>> Fido.search(a.Time(date_start, date_end), a.Instrument('adapt'), a.adapt.ADAPTLonType(longitude_type)) # doctest: +REMOTE_DATA + + Results from 1 Provider: + + 10 Results from the ADAPTClient: + + Start Time End Time Instrument Provider Source ... ADAPTMagData days_since_last_obs hours_since_last_obs minutes_since_last_obs seconds_since_last_obs + ----------------------- ----------------------- ---------- -------- ------ ... ------------ ------------------- -------------------- ---------------------- ---------------------- + 2017-07-20 08:00:00.000 2017-07-20 08:00:59.999 ADAPT NSO GONG ... 1 0 1 56 0 + 2017-07-20 14:00:00.000 2017-07-20 14:00:59.999 ADAPT NSO GONG ... 1 0 1 56 0 + 2017-07-20 20:00:00.000 2017-07-20 20:00:59.999 ADAPT NSO GONG ... 1 0 1 56 0 + 2017-07-21 02:00:00.000 2017-07-21 02:00:59.999 ADAPT NSO GONG ... 1 0 1 56 0 + 2017-07-21 08:00:00.000 2017-07-21 08:00:59.999 ADAPT NSO GONG ... 1 0 1 56 0 + 2017-07-21 14:00:00.000 2017-07-21 14:00:59.999 ADAPT NSO GONG ... 1 0 1 56 0 + 2017-07-21 20:00:00.000 2017-07-21 20:00:59.999 ADAPT NSO GONG ... 1 0 1 56 0 + 2017-07-22 02:00:00.000 2017-07-22 02:00:59.999 ADAPT NSO GONG ... 1 0 1 56 0 + 2017-07-22 08:00:00.000 2017-07-22 08:00:59.999 ADAPT NSO GONG ... 1 0 4 36 0 + 2017-07-22 14:00:00.000 2017-07-22 14:00:59.999 ADAPT NSO GONG ... 1 0 1 56 0 + + + + References + ---------- + `Names and possible attrs values are available `__. + """ + baseurl = r'https://gong.nso.edu/adapt/maps/gong/%Y/adapt(\d){5}_(\d){2}(\w){1}(\d){3}_(\d){12}_(\w){1}(\d){8}(\w){1}(\d){1}\.fts\.gz' + pattern = '{}adapt{ADAPTFileType:1d}{ADAPTLonType:1d}{ADAPTInputSource:1d}{ADAPTDataAssimilation:1d}{ADAPTResolution:1d}' + \ + '_{ADAPTVersionYear:2d}{ADAPTVersionMonth:1l}{ADAPTRealizations:3d}_{year:4d}{month:2d}{day:2d}{hour:2d}{minute:2d}' + \ + '_{ADAPTEvolutionMode:1l}{days_since_last_obs:2d}{hours_since_last_obs:2d}{minutes_since_last_obs:2d}{seconds_since_last_obs:2d}{ADAPTHelioData:1l}{ADAPTMagData:1d}.fts.gz' + + @classmethod + def _attrs_module(cls): + return 'adapt', 'sunpy.net.dataretriever.attrs.adapt' + + @classmethod + def register_values(cls): + adict = { + a.Instrument: [('ADAPT', 'ADvanced Adaptive Prediction Technique.')], + a.Provider: [('NSO', 'National Solar Observatory.')], + a.Source: [('GONG', 'Global Oscillation Network Group.')], + ADAPTFileType: [('4', 'Public')], + ADAPTLonType: [('0', 'Carrington Fixed'), ('1', 'Central Meridian'), ('2', 'East Limb')], + ADAPTInputSource: [('0', 'All'), ('1', 'KPVT'), ('2', 'VSM'), ('3', 'GONG'), ('4', 'HMI'), ('5', 'MDI'), ('6', 'MWO')], + ADAPTDataAssimilation: [('0', 'WH'), ('1', 'enLS'), ('2', 'enkf'), ('3', 'enLAKF')], + ADAPTResolution: [('1', '1.0 deg'), ('2', '0.2 deg')], + ADAPTVersionYear: [(str(i), f"Code version year -> {2000 + i}") for i in range(1, 20)], + ADAPTRealizations: [(str(i), f"Number of Realizations -> {i}") for i in range(1, 20)], + ADAPTVersionMonth: [(chr(i+96), f"Code version month -> {i}") for i in range(1, 13)], + ADAPTEvolutionMode: [('a', 'Data assimilation step'), ('i', 'Intermediate step'), ('f', 'Forecast step')], + ADAPTHelioData: [('n', 'Not added or no data'), ('f', 'Far-side'), ('e', 'Emergence'), ('b', 'Both emergence & far-side')], + ADAPTMagData: [('0', 'Not added or no data'), ('1', 'Mag-los'), ('2', 'Mag-vector'), ('3', 'Mag- both los & vector'), + ('4', 'Mag- polar avg obs'), ('5', 'Mag- los & polar'), ('6', 'Mag- vector & polar'), ('7', 'Mag- both los and vector & polar')] + } + return adict diff --git a/sunpy/net/dataretriever/sources/tests/test_adapt.py b/sunpy/net/dataretriever/sources/tests/test_adapt.py new file mode 100644 index 00000000000..6d14bc0881a --- /dev/null +++ b/sunpy/net/dataretriever/sources/tests/test_adapt.py @@ -0,0 +1,104 @@ + +import tempfile + +import pytest +from hypothesis import given + +import sunpy.net.dataretriever.sources.adapt as adapt +from sunpy.net import attrs as a +from sunpy.net.dataretriever.client import QueryResponse +from sunpy.net.tests.strategies import time_attr +from sunpy.time import parse_time + + +@pytest.fixture +def adapt_client(): + return adapt.ADAPTClient() + + +@given(time_attr()) +def test_can_handle_query(time): + # Hypothesis complains if we use the fixture + adapt_client = adapt.ADAPTClient() + ans1 = adapt_client._can_handle_query(time, a.Instrument.adapt) + assert ans1 is True + ans2 = adapt_client._can_handle_query(time, a.Instrument.adapt, + a.adapt.ADAPTResolution('1')) + assert ans2 is True + ans3 = adapt_client._can_handle_query(time, a.Instrument.adapt, + a.adapt.ADAPTResolution('1'), + a.adapt.ADAPTHelioData('f')) + assert ans3 is True + ans4 = adapt_client._can_handle_query(time) + assert ans4 is False + ans5 = adapt_client._can_handle_query(time, a.Instrument.adapt, a.Provider.nso) + assert ans5 is True + ans6 = adapt_client._can_handle_query(time, a.Instrument.adapt, + a.adapt.ADAPTLonType('0')) + assert ans6 is True + + +def mock_query_object(adapt_client): + """ + Creating a Query Response object and prefilling it with some information + """ + start = '2019-05-25T02:00:00.00' + end = '2019-05-25T02:00:59.999' + obj = { + 'Start Time': parse_time(start), + 'End Time': parse_time(end), + 'Instrument': 'ADAPT', + 'Physobs': 'flux', + 'Source': 'GONG', + 'Provider': 'NSO', + 'url': ("https://gong.nso.edu/adapt/maps/gong/2019/adapt40311_03i012_201905250200_i00005600n0.fts.gz") + } + results = QueryResponse([obj], client=adapt_client) + return results + + +@pytest.mark.remote_data +def test_fetch_working(adapt_client): + """ + Tests if the online server is working. + This also checks if the mock is working well. + """ + start = '2019/05/25 02:00:00' + end = '2019/05/26 02:00:59.999' + tr = a.Time(start, end) + qr = adapt_client.search(tr, a.Instrument.adapt)[0] + mock_qr = mock_query_object(adapt_client)[0] + + assert mock_qr['Source'] == qr['Source'] + assert mock_qr['Provider'] == qr['Provider'] + assert mock_qr['Instrument'] == qr['Instrument'] + assert mock_qr['url'] == qr['url'] + assert qr['Start Time'].isot == mock_qr['Start Time'].isot + assert qr['End Time'].isot == mock_qr['End Time'].isot + + with tempfile.TemporaryDirectory() as tmpdirname: + download_list = adapt_client.fetch(qr, path=tmpdirname) + assert len(download_list) == 1 + + + +def test_show(adapt_client): + mock_qr = mock_query_object(adapt_client) + qrshow0 = mock_qr.show() + qrshow1 = mock_qr.show('Start Time', 'Instrument') + allcols = {'Start Time', 'End Time', 'Instrument', 'Source', 'Provider', 'url'} + assert not allcols.difference(qrshow0.colnames) + assert qrshow1.colnames == ['Start Time', 'Instrument'] + assert qrshow0['Instrument'][0] == 'ADAPT' + + +def test_attr_reg(): + assert a.Instrument.adapt == a.Instrument('ADAPT') + + +def test_client_repr(adapt_client): + """ + Repr check + """ + output = str(adapt_client) + assert output[:50] == 'sunpy.net.dataretriever.sources.adapt.ADAPTClient\n'