From a0d716c17309abd0caf4fa99352ec95ca2520691 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 12 Oct 2021 15:30:32 -0400 Subject: [PATCH 01/18] add Fido client and filetype attr --- eispac/net/__init__.py | 3 ++ eispac/net/attrs.py | 22 +++++++++++++ eispac/net/client.py | 74 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 eispac/net/__init__.py create mode 100644 eispac/net/attrs.py create mode 100644 eispac/net/client.py diff --git a/eispac/net/__init__.py b/eispac/net/__init__.py new file mode 100644 index 0000000..ea0f4ca --- /dev/null +++ b/eispac/net/__init__.py @@ -0,0 +1,3 @@ +from .client import * + +__all__ = ['EISClient'] diff --git a/eispac/net/attrs.py b/eispac/net/attrs.py new file mode 100644 index 0000000..14a66b4 --- /dev/null +++ b/eispac/net/attrs.py @@ -0,0 +1,22 @@ +from sunpy.net.attr import SimpleAttr + +__all__ = ['FileType'] + + +class FileType(SimpleAttr): + """ + Specifies the type of EIS level 1 file + + Parameters + ---------- + value: `str` + Either "data" or "head" + """ + + def __init__(self, value): + if not isinstance(value, str): + raise ValueError('File type must be a string') + value = value.lower() + if value not in ['data', 'head']: + raise ValueError(f'File type {value} must be either "data" or "head".') + super().__init__(value) diff --git a/eispac/net/client.py b/eispac/net/client.py new file mode 100644 index 0000000..fcff750 --- /dev/null +++ b/eispac/net/client.py @@ -0,0 +1,74 @@ +from sunpy.net.dataretriever import GenericClient + +from eispac.net.attrs import FileType + +__all__ = ['EISClient'] + + +class EISClient(GenericClient): + """ + Provides access to the level 1 EIS data in HDF5 format. + + This data is hosted by the `Naval Research Laboratory `__ + + Examples + -------- + >>> from sunpy.net import Fido, attrs as a + >>> import eispac.net + >>> from eispac.net.attrs import FileType + >>> results = Fido.search(a.Time('2020-11-09 00:00:00','2020-11-09 01:00:00'), + ... a.Instrument('EIS'), + ... a.Physobs.intensity, + ... a.Source('Hinode'), + ... a.Provider('NRL'), + ... a.Level('1')) + >>> results + + Results from 1 Provider: + + 2 Results from the EISClient: + Start Time End Time Instrument Physobs Source Provider Level FileType + ----------------------- ----------------------- ---------- --------- ------ -------- ----- -------- + 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS intensity HINODE NRL 1 data + 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS intensity HINODE NRL 1 head + + + >>> results = Fido.search(a.Time('2020-11-09 00:00:00','2020-11-09 01:00:00'), + ... a.Instrument('EIS'), + ... a.Physobs.intensity, + ... a.Source('Hinode'), + ... a.Provider('NRL'), + ... a.Level('1'), + ... FileType('head')) + >>> results + + Results from 1 Provider: + + 1 Results from the EISClient: + Start Time End Time Instrument Physobs Source Provider Level FileType + ----------------------- ----------------------- ---------- --------- ------ -------- ----- -------- + 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS intensity HINODE NRL 1 head + + + """ + baseurl = r'https://eis.nrl.navy.mil/level1/hdf5/%Y/%m/%d/eis_%Y%m%d_%H%M%S.(\w){4}.h5' + pattern = '{}/hdf5/{year:4d}/{month:2d}/{day:2d}/eis_{:8d}_{hour:2d}{minute:2d}{second:2d}.{FileType}.{}' + + @property + def info_url(self): + return 'https://eis.nrl.navy.mil/' + + @classmethod + def register_values(cls): + from sunpy.net import attrs + return { + attrs.Instrument: [('EIS', 'Extreme Ultraviolet Imaging Spectrometer')], + attrs.Physobs: [('intensity', 'Spectrally resolved intensity in detector units')], + attrs.Source: [('Hinode', 'The Hinode mission is a partnership between JAXA, NASA, and UKSA')], + attrs.Provider: [('NRL', 'U.S. Naval Research Laboratory')], + attrs.Level: [ + ('1', 'EIS: The EIS client can only return level 1 data. Level 0 EIS data is available from the VSO.') + ], + FileType: [('data', 'These files contain the actual intensity data'), + ('head', 'These files contain only the header metadata')], + } From 4291f1d9bc23f493599c0dc926466a39bfec3474 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 12 Oct 2021 15:39:21 -0400 Subject: [PATCH 02/18] example formatting --- eispac/net/client.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/eispac/net/client.py b/eispac/net/client.py index fcff750..0b8ca5d 100644 --- a/eispac/net/client.py +++ b/eispac/net/client.py @@ -9,7 +9,7 @@ class EISClient(GenericClient): """ Provides access to the level 1 EIS data in HDF5 format. - This data is hosted by the `Naval Research Laboratory `__ + This data is hosted by the `Naval Research Laboratory `__. Examples -------- @@ -21,9 +21,9 @@ class EISClient(GenericClient): ... a.Physobs.intensity, ... a.Source('Hinode'), ... a.Provider('NRL'), - ... a.Level('1')) - >>> results - + ... a.Level('1')) #doctest: +REMOTE_DATA + >>> results #doctest: +REMOTE_DATA + Results from 1 Provider: 2 Results from the EISClient: @@ -39,9 +39,9 @@ class EISClient(GenericClient): ... a.Source('Hinode'), ... a.Provider('NRL'), ... a.Level('1'), - ... FileType('head')) - >>> results - + ... FileType('head')) #doctest: +REMOTE_DATA + >>> results #doctest: +REMOTE_DATA + Results from 1 Provider: 1 Results from the EISClient: From 403e9719e8b5bcbb9c878a99696b038839750b3a Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Thu, 14 Oct 2021 17:00:38 -0400 Subject: [PATCH 03/18] Input for filetype is now header instead of head --- eispac/net/attrs.py | 8 +++++--- eispac/net/client.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/eispac/net/attrs.py b/eispac/net/attrs.py index 14a66b4..df89488 100644 --- a/eispac/net/attrs.py +++ b/eispac/net/attrs.py @@ -10,13 +10,15 @@ class FileType(SimpleAttr): Parameters ---------- value: `str` - Either "data" or "head" + Either "data" or "header" """ def __init__(self, value): if not isinstance(value, str): raise ValueError('File type must be a string') value = value.lower() - if value not in ['data', 'head']: - raise ValueError(f'File type {value} must be either "data" or "head".') + if value not in ['data', 'header']: + raise ValueError(f'File type {value} must be either "data" or "header".') + # The actual files are labeled "head" but "header" is more readable + value = 'head' if value == 'header' else value super().__init__(value) diff --git a/eispac/net/client.py b/eispac/net/client.py index 0b8ca5d..10246e8 100644 --- a/eispac/net/client.py +++ b/eispac/net/client.py @@ -70,5 +70,5 @@ def register_values(cls): ('1', 'EIS: The EIS client can only return level 1 data. Level 0 EIS data is available from the VSO.') ], FileType: [('data', 'These files contain the actual intensity data'), - ('head', 'These files contain only the header metadata')], + ('header', 'These files contain only the header metadata')], } From cd00f8a48d715e07f9d8733fd9d9410b42442d7c Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 12 Oct 2021 15:30:32 -0400 Subject: [PATCH 04/18] add Fido client and filetype attr --- eispac/net/attrs.py | 6 ++---- eispac/net/client.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/eispac/net/attrs.py b/eispac/net/attrs.py index df89488..207134e 100644 --- a/eispac/net/attrs.py +++ b/eispac/net/attrs.py @@ -17,8 +17,6 @@ def __init__(self, value): if not isinstance(value, str): raise ValueError('File type must be a string') value = value.lower() - if value not in ['data', 'header']: - raise ValueError(f'File type {value} must be either "data" or "header".') - # The actual files are labeled "head" but "header" is more readable - value = 'head' if value == 'header' else value + if value not in ['data', 'head']: + raise ValueError(f'File type {value} must be either "data" or "head".') super().__init__(value) diff --git a/eispac/net/client.py b/eispac/net/client.py index 10246e8..2699077 100644 --- a/eispac/net/client.py +++ b/eispac/net/client.py @@ -9,7 +9,11 @@ class EISClient(GenericClient): """ Provides access to the level 1 EIS data in HDF5 format. +<<<<<<< HEAD This data is hosted by the `Naval Research Laboratory `__. +======= + This data is hosted by the `Naval Research Laboratory `__ +>>>>>>> add Fido client and filetype attr Examples -------- @@ -21,9 +25,15 @@ class EISClient(GenericClient): ... a.Physobs.intensity, ... a.Source('Hinode'), ... a.Provider('NRL'), +<<<<<<< HEAD ... a.Level('1')) #doctest: +REMOTE_DATA >>> results #doctest: +REMOTE_DATA +======= + ... a.Level('1')) + >>> results + +>>>>>>> add Fido client and filetype attr Results from 1 Provider: 2 Results from the EISClient: From 95b2408257e7f57d014787806ddd37c895133e74 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 12 Oct 2021 15:39:21 -0400 Subject: [PATCH 05/18] example formatting --- eispac/net/client.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/eispac/net/client.py b/eispac/net/client.py index 2699077..fd1f400 100644 --- a/eispac/net/client.py +++ b/eispac/net/client.py @@ -9,11 +9,15 @@ class EISClient(GenericClient): """ Provides access to the level 1 EIS data in HDF5 format. +<<<<<<< HEAD <<<<<<< HEAD This data is hosted by the `Naval Research Laboratory `__. ======= This data is hosted by the `Naval Research Laboratory `__ >>>>>>> add Fido client and filetype attr +======= + This data is hosted by the `Naval Research Laboratory `__. +>>>>>>> example formatting Examples -------- @@ -25,15 +29,9 @@ class EISClient(GenericClient): ... a.Physobs.intensity, ... a.Source('Hinode'), ... a.Provider('NRL'), -<<<<<<< HEAD ... a.Level('1')) #doctest: +REMOTE_DATA >>> results #doctest: +REMOTE_DATA -======= - ... a.Level('1')) - >>> results - ->>>>>>> add Fido client and filetype attr Results from 1 Provider: 2 Results from the EISClient: From 999383bcad62382fa3c0e492de3cc449ea033948 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Thu, 14 Oct 2021 17:00:38 -0400 Subject: [PATCH 06/18] Input for filetype is now header instead of head --- eispac/net/attrs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eispac/net/attrs.py b/eispac/net/attrs.py index 207134e..df89488 100644 --- a/eispac/net/attrs.py +++ b/eispac/net/attrs.py @@ -17,6 +17,8 @@ def __init__(self, value): if not isinstance(value, str): raise ValueError('File type must be a string') value = value.lower() - if value not in ['data', 'head']: - raise ValueError(f'File type {value} must be either "data" or "head".') + if value not in ['data', 'header']: + raise ValueError(f'File type {value} must be either "data" or "header".') + # The actual files are labeled "head" but "header" is more readable + value = 'head' if value == 'header' else value super().__init__(value) From 13f12d5e84d6a736ac73f9e5aab6ab75ade00f40 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 2 Nov 2021 13:21:03 -0400 Subject: [PATCH 07/18] add pytest-doctest dependency --- setup.cfg | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/setup.cfg b/setup.cfg index e8e9494..e6f2e3b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,6 +39,7 @@ install_requires = [options.extras_require] test = pytest>=4.6.3 + pytest-astropy docs = sphinx==4.0.2 # why is this pinned so strictly? sphinx-automodapi>=0.13 @@ -47,3 +48,12 @@ docs = [options.package_data] eispac.data.test = *.h5 eispac.data.templates = *.template.* + +[tool:pytest] +testpaths = "eispac" "docs" +norecursedirs = ".tox" "build" "docs[\/]_build" "docs[\/]generated" "*.egg-info" "examples" ".jupyter" ".history" "tools" +doctest_plus = enabled +text_file_format = rst +addopts = --doctest-rst +doctest_optionflags = NORMALIZE_WHITESPACE FLOAT_CMP ELLIPSIS +remote_data_strict = True From 47f289797628cfbbc680be3cbf4f536765718232 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 2 Nov 2021 13:21:24 -0400 Subject: [PATCH 08/18] docs for net module --- docs/code_reference/eispac_net.rst | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/code_reference/eispac_net.rst diff --git a/docs/code_reference/eispac_net.rst b/docs/code_reference/eispac_net.rst new file mode 100644 index 0000000..129b596 --- /dev/null +++ b/docs/code_reference/eispac_net.rst @@ -0,0 +1,8 @@ +eispc net +========= + +.. automodapi:: eispac.net + :no-heading: + +.. automodapi:: eispac.net.attrs + :headings: ^" From dfed4762f181a9baea1e58ce934839e5621599b8 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 2 Nov 2021 13:21:47 -0400 Subject: [PATCH 09/18] go back to head; fix doctests --- eispac/net/attrs.py | 6 ++---- eispac/net/client.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/eispac/net/attrs.py b/eispac/net/attrs.py index df89488..207134e 100644 --- a/eispac/net/attrs.py +++ b/eispac/net/attrs.py @@ -17,8 +17,6 @@ def __init__(self, value): if not isinstance(value, str): raise ValueError('File type must be a string') value = value.lower() - if value not in ['data', 'header']: - raise ValueError(f'File type {value} must be either "data" or "header".') - # The actual files are labeled "head" but "header" is more readable - value = 'head' if value == 'header' else value + if value not in ['data', 'head']: + raise ValueError(f'File type {value} must be either "data" or "head".') super().__init__(value) diff --git a/eispac/net/client.py b/eispac/net/client.py index fd1f400..7e1318e 100644 --- a/eispac/net/client.py +++ b/eispac/net/client.py @@ -35,10 +35,10 @@ class EISClient(GenericClient): Results from 1 Provider: 2 Results from the EISClient: - Start Time End Time Instrument Physobs Source Provider Level FileType - ----------------------- ----------------------- ---------- --------- ------ -------- ----- -------- - 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS intensity HINODE NRL 1 data - 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS intensity HINODE NRL 1 head + Start Time End Time Instrument ... Level FileType + ----------------------- ----------------------- ---------- ... ----- -------- + 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS ... 1 data + 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS ... 1 head >>> results = Fido.search(a.Time('2020-11-09 00:00:00','2020-11-09 01:00:00'), @@ -53,9 +53,9 @@ class EISClient(GenericClient): Results from 1 Provider: 1 Results from the EISClient: - Start Time End Time Instrument Physobs Source Provider Level FileType - ----------------------- ----------------------- ---------- --------- ------ -------- ----- -------- - 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS intensity HINODE NRL 1 head + Start Time End Time Instrument ... Level FileType + ----------------------- ----------------------- ---------- ... ----- -------- + 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS ... 1 head """ @@ -77,6 +77,6 @@ def register_values(cls): attrs.Level: [ ('1', 'EIS: The EIS client can only return level 1 data. Level 0 EIS data is available from the VSO.') ], - FileType: [('data', 'These files contain the actual intensity data'), - ('header', 'These files contain only the header metadata')], + FileType: [('data', 'These files contain the actual intensity data.'), + ('head', 'These files contain only the header metadata.')], } From d015f796a88ad2b82b33c01550205119c316cdb2 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 2 Nov 2021 13:22:03 -0400 Subject: [PATCH 10/18] run remote tests on the CI --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 65f2a1c..93c17b7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,7 +27,7 @@ jobs: run: | pip install -e .[test,docs] - name: Test - run: pytest + run: pytest --remote-data=any - name: Build docs run: | cd docs From e77befa6b369cc2637c971f7427e71966c0ca89a Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 2 Nov 2021 13:29:55 -0400 Subject: [PATCH 11/18] cleanup docstring --- eispac/net/client.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/eispac/net/client.py b/eispac/net/client.py index 7e1318e..f1d1f80 100644 --- a/eispac/net/client.py +++ b/eispac/net/client.py @@ -9,15 +9,7 @@ class EISClient(GenericClient): """ Provides access to the level 1 EIS data in HDF5 format. -<<<<<<< HEAD -<<<<<<< HEAD This data is hosted by the `Naval Research Laboratory `__. -======= - This data is hosted by the `Naval Research Laboratory `__ ->>>>>>> add Fido client and filetype attr -======= - This data is hosted by the `Naval Research Laboratory `__. ->>>>>>> example formatting Examples -------- From ece927c90202e38d6cd23e1b08f3500340954677 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 2 Nov 2021 13:32:03 -0400 Subject: [PATCH 12/18] add need optional dependencies for map --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e6f2e3b..f1fb768 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,7 @@ install_requires = matplotlib>=3.1 h5py>=2.9 astropy>=3.1 - sunpy[map]>=2.1 + sunpy[net,map]>=2.1 ndcube>=2.0.0 parfive>=1.5 python-dateutil>=2.8 From 0cd77988910e923ca48dc7e7735e3a60826a0fbb Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 2 Nov 2021 13:50:33 -0400 Subject: [PATCH 13/18] fix rot_xy doctest; dont test docs for now --- eispac/util/rot_xy.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eispac/util/rot_xy.py b/eispac/util/rot_xy.py index 4afbd66..3055113 100644 --- a/eispac/util/rot_xy.py +++ b/eispac/util/rot_xy.py @@ -33,7 +33,7 @@ def rot_xy(xcen, ycen, start_time, end_time): Examples -------- - >>> new = rot_xy(0, 0, start_time='01-JAN-2021 00:00', end_time='01-JAN-2021 01:00') + >>> new = rot_xy(0, 0, start_time='2021-JAN-01 00:00', end_time='2021-JAN-01 01:00') >>> print(new.Tx, new.Ty) 9.47188arcsec 0.0809565arcsec """ diff --git a/setup.cfg b/setup.cfg index f1fb768..a9d22f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,7 +50,7 @@ eispac.data.test = *.h5 eispac.data.templates = *.template.* [tool:pytest] -testpaths = "eispac" "docs" +testpaths = "eispac" norecursedirs = ".tox" "build" "docs[\/]_build" "docs[\/]generated" "*.egg-info" "examples" ".jupyter" ".history" "tools" doctest_plus = enabled text_file_format = rst From 27a7ed8d1aa567f981f382bd4d09de68655d1dfe Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 25 Oct 2022 17:15:11 -0400 Subject: [PATCH 14/18] add client tests --- eispac/net/tests/test_eis_client.py | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 eispac/net/tests/test_eis_client.py diff --git a/eispac/net/tests/test_eis_client.py b/eispac/net/tests/test_eis_client.py new file mode 100644 index 0000000..a739259 --- /dev/null +++ b/eispac/net/tests/test_eis_client.py @@ -0,0 +1,50 @@ +import pytest +from sunpy.net import Fido, attrs as a +import eispac.net + + +@pytest.fixture +def eis_query(): + time = a.Time('2022-03-29 22:21:00','2022-03-29 23:21:00') + instr = a.Instrument('EIS') + obs = a.Physobs.intensity + source = a.Source('Hinode') + provider = a.Provider('NRL') + level = a.Level('1') + return time, instr, obs, source, provider, level + + +@pytest.mark.remote_data +def test_search_all_types(eis_query): + q = Fido.search(*eis_query) + assert len(q) == 1 + assert len(q[0]) == 3 + assert q[0,0]['url'] == 'https://eis.nrl.navy.mil/level1/hdf5/2022/03/29/eis_20220329_222113.data.h5' + + +@pytest.mark.remote_data +def test_search_fits_only(eis_query): + q = Fido.search(*eis_query, a.eispac.FileType('FITS')) + assert len(q) == 1 + assert len(q[0]) == 1 + assert q[0,0]['url'] == 'https://eis.nrl.navy.mil/level1/fits/2022/03/29/eis_er_20220329_222113.fits' + + +@pytest.mark.parametrize('file_type, file_url', [ + ('FITS', 'https://eis.nrl.navy.mil/level1/fits/2022/03/29/eis_er_20220329_222113.fits'), + ('HDF5 data', 'https://eis.nrl.navy.mil/level1/hdf5/2022/03/29/eis_20220329_222113.data.h5'), + ('HDF5 header', 'https://eis.nrl.navy.mil/level1/hdf5/2022/03/29/eis_20220329_222113.head.h5') +]) +@pytest.mark.remote_data +def test_search_individual_filetypes(eis_query, file_type, file_url): + q = Fido.search(*eis_query, a.eispac.FileType(file_type)) + assert len(q) == 1 + assert len(q[0]) == 1 + assert q[0,0]['url'] == file_url + assert q[0,0]['FileType'] == file_type + + +def test_registered_attrs(): + attr_names = ['fits', 'data_h5', 'head_h5'] + for an in attr_names: + assert hasattr(a.eispac.FileType, an) From 6898bda8ac27f587e25415954c50b638236a3514 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 25 Oct 2022 17:15:41 -0400 Subject: [PATCH 15/18] update client to include FITS files, more intuitive attrs search --- eispac/net/attrs.py | 12 +++-- eispac/net/client.py | 103 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 91 insertions(+), 24 deletions(-) diff --git a/eispac/net/attrs.py b/eispac/net/attrs.py index 207134e..09fa26e 100644 --- a/eispac/net/attrs.py +++ b/eispac/net/attrs.py @@ -10,13 +10,19 @@ class FileType(SimpleAttr): Parameters ---------- value: `str` - Either "data" or "header" + Possible values are "HDF5 data" or "HDF5 header" to retrieve the + data and header files, respectively, in HDF5 format, or "FITS" to + retrieve the FITS files. Inputs are not case sensitive. """ def __init__(self, value): if not isinstance(value, str): raise ValueError('File type must be a string') value = value.lower() - if value not in ['data', 'head']: - raise ValueError(f'File type {value} must be either "data" or "head".') + if 'hdf5' in value: + value = '.'.join([value[5:], 'h5']) + if value == 'header.h5': + value = 'head.h5' + if value not in ['data.h5', 'head.h5', 'fits']: + raise ValueError(f'File type {value} must be either "HDF5 data", "HDF5 header", or "FITS".') super().__init__(value) diff --git a/eispac/net/client.py b/eispac/net/client.py index f1d1f80..6d8f2d0 100644 --- a/eispac/net/client.py +++ b/eispac/net/client.py @@ -1,4 +1,7 @@ -from sunpy.net.dataretriever import GenericClient +from sunpy.net import attrs as a +from sunpy.net.dataretriever import GenericClient, QueryResponse +from sunpy.net.scraper import Scraper +from sunpy.time import TimeRange from eispac.net.attrs import FileType @@ -7,7 +10,7 @@ class EISClient(GenericClient): """ - Provides access to the level 1 EIS data in HDF5 format. + Provides access to the level 1 EIS data in HDF5 and FITS format. This data is hosted by the `Naval Research Laboratory `__. @@ -26,11 +29,14 @@ class EISClient(GenericClient): Results from 1 Provider: - 2 Results from the EISClient: - Start Time End Time Instrument ... Level FileType - ----------------------- ----------------------- ---------- ... ----- -------- - 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS ... 1 data - 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS ... 1 head + 3 Results from the EISClient: + Source: https://eis.nrl.navy.mil/ + + Start Time End Time ... Level FileType + ----------------------- ----------------------- ... ----- ----------- + 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 ... 1 HDF5 data + 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 ... 1 HDF5 header + 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 ... 1 FITS >>> results = Fido.search(a.Time('2020-11-09 00:00:00','2020-11-09 01:00:00'), @@ -39,36 +45,91 @@ class EISClient(GenericClient): ... a.Source('Hinode'), ... a.Provider('NRL'), ... a.Level('1'), - ... FileType('head')) #doctest: +REMOTE_DATA + ... FileType('HDF5 header')) #doctest: +REMOTE_DATA >>> results #doctest: +REMOTE_DATA Results from 1 Provider: 1 Results from the EISClient: - Start Time End Time Instrument ... Level FileType - ----------------------- ----------------------- ---------- ... ----- -------- - 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 EIS ... 1 head + Source: https://eis.nrl.navy.mil/ + + Start Time End Time ... Level FileType + ----------------------- ----------------------- ... ----- ----------- + 2020-11-09 00:10:12.000 2020-11-09 00:10:12.999 ... 1 HDF5 header """ - baseurl = r'https://eis.nrl.navy.mil/level1/hdf5/%Y/%m/%d/eis_%Y%m%d_%H%M%S.(\w){4}.h5' - pattern = '{}/hdf5/{year:4d}/{month:2d}/{day:2d}/eis_{:8d}_{hour:2d}{minute:2d}{second:2d}.{FileType}.{}' + baseurl_hdf5 = r'https://eis.nrl.navy.mil/level1/hdf5/%Y/%m/%d/eis_%Y%m%d_%H%M%S.(\w){4}.h5' + pattern_hdf5 = '{}/{year:4d}/{month:2d}/{day:2d}/eis_{:8d}_{hour:2d}{minute:2d}{second:2d}.{FileType}' + baseurl_fits = r'https://eis.nrl.navy.mil/level1/fits/%Y/%m/%d/eis_er_%Y%m%d_%H%M%S.fits' + pattern_fits = '{}/{year:4d}/{month:2d}/{day:2d}/eis_er_{:8d}_{hour:2d}{minute:2d}{second:2d}.{FileType}' @property def info_url(self): return 'https://eis.nrl.navy.mil/' + + def search(self, *args, **kwargs): + metalist = [] + matchdict = self._get_match_dict(*args, **kwargs) + all_filetypes = matchdict.get('FileType') + for ft in all_filetypes: + if 'h5' in ft: + baseurl = self.baseurl_hdf5 + pattern = self.pattern_hdf5 + else: + baseurl = self.baseurl_fits + pattern = self.pattern_fits + + scraper = Scraper(baseurl, regex=True) + tr = TimeRange(matchdict['Start Time'], matchdict['End Time']) + filesmeta = scraper._extract_files_meta(tr, extractor=pattern, matcher={'FileType': ft}) + filesmeta = sorted(filesmeta, key=lambda k: k['url']) + for i in filesmeta: + rowdict = self.post_search_hook(i, matchdict) + metalist.append(rowdict) + + return QueryResponse(metalist, client=self) + + def post_search_hook(self, i, matchdict): + # This makes the final display names of the file types nicer + filetype_mapping = { + 'data.h5': 'HDF5 data', + 'head.h5': 'HDF5 header', + 'fits': 'FITS', + } + rd = super().post_search_hook(i, matchdict) + rd['FileType'] = filetype_mapping[rd['FileType']] + return rd @classmethod def register_values(cls): - from sunpy.net import attrs return { - attrs.Instrument: [('EIS', 'Extreme Ultraviolet Imaging Spectrometer')], - attrs.Physobs: [('intensity', 'Spectrally resolved intensity in detector units')], - attrs.Source: [('Hinode', 'The Hinode mission is a partnership between JAXA, NASA, and UKSA')], - attrs.Provider: [('NRL', 'U.S. Naval Research Laboratory')], - attrs.Level: [ + a.Instrument: [('EIS', 'Extreme Ultraviolet Imaging Spectrometer')], + a.Physobs: [('intensity', 'Spectrally resolved intensity in detector units')], + a.Source: [('Hinode', 'The Hinode mission is a partnership between JAXA, NASA, and UKSA')], + a.Provider: [('NRL', 'U.S. Naval Research Laboratory')], + a.Level: [ ('1', 'EIS: The EIS client can only return level 1 data. Level 0 EIS data is available from the VSO.') ], - FileType: [('data', 'These files contain the actual intensity data.'), - ('head', 'These files contain only the header metadata.')], + FileType: [('data.h5', 'These files contain the actual intensity data in HDF5 format.'), + ('head.h5', 'These files contain only the header metadata in HDF5 format.'), + ('fits', 'These files contain both data and metadata in FITS format')], } + + @classmethod + def _attrs_module(cls): + # Register EIS specific attributes with Fido + return 'eispac', 'eispac.net.attrs' + + @classmethod + def _can_handle_query(cls, *query): + """ + Check if this client can handle a given Fido query. + Returns + ------- + bool + True if this client can handle the given query. + """ + required = {a.Time, a.Instrument, a.Source} + optional = {a.Provider, a.Physobs, a.Level, FileType} + return cls.check_attr_types_in_query(query, required, optional) From e92a99e3a9b1adaa36fc05f76d9d6f26d2fd9232 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 25 Oct 2022 17:19:59 -0400 Subject: [PATCH 16/18] one more test --- eispac/net/tests/test_eis_client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/eispac/net/tests/test_eis_client.py b/eispac/net/tests/test_eis_client.py index a739259..6740028 100644 --- a/eispac/net/tests/test_eis_client.py +++ b/eispac/net/tests/test_eis_client.py @@ -44,6 +44,16 @@ def test_search_individual_filetypes(eis_query, file_type, file_url): assert q[0,0]['FileType'] == file_type +@pytest.mark.remote_data +def test_combined_hdf5_search(eis_query): + q = Fido.search(*eis_query, + a.eispac.FileType('HDF5 data') | a.eispac.FileType('HDF5 header')) + assert len(q) == 1 + assert len(q[0]) == 2 + assert q[0,0]['FileType'] == 'HDF5 data' + assert q[0,1]['FileType'] == 'HDF5 header' + + def test_registered_attrs(): attr_names = ['fits', 'data_h5', 'head_h5'] for an in attr_names: From 66081b9054889032322487a0d5fcafc7733941fd Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 25 Oct 2022 17:24:58 -0400 Subject: [PATCH 17/18] fix the test --- eispac/net/tests/test_eis_client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/eispac/net/tests/test_eis_client.py b/eispac/net/tests/test_eis_client.py index 6740028..8d51374 100644 --- a/eispac/net/tests/test_eis_client.py +++ b/eispac/net/tests/test_eis_client.py @@ -48,10 +48,11 @@ def test_search_individual_filetypes(eis_query, file_type, file_url): def test_combined_hdf5_search(eis_query): q = Fido.search(*eis_query, a.eispac.FileType('HDF5 data') | a.eispac.FileType('HDF5 header')) - assert len(q) == 1 - assert len(q[0]) == 2 + assert len(q) == 2 + assert len(q[0]) == 1 + assert len(q[1]) == 1 assert q[0,0]['FileType'] == 'HDF5 data' - assert q[0,1]['FileType'] == 'HDF5 header' + assert q[1,0]['FileType'] == 'HDF5 header' def test_registered_attrs(): From 59eb1a2d5bb6df95e5a9e791504fb29c2a27075d Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Wed, 26 Oct 2022 09:25:26 -0400 Subject: [PATCH 18/18] clarifying comment --- eispac/net/client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eispac/net/client.py b/eispac/net/client.py index 6d8f2d0..fe154b3 100644 --- a/eispac/net/client.py +++ b/eispac/net/client.py @@ -69,6 +69,8 @@ def info_url(self): return 'https://eis.nrl.navy.mil/' def search(self, *args, **kwargs): + # NOTE: Search is overridden because URL and pattern depending on the filetype. + # This enables multiple filetypes to be returned in the same query. metalist = [] matchdict = self._get_match_dict(*args, **kwargs) all_filetypes = matchdict.get('FileType')