Merge pull request #2916 from ericpre/fix_eelsdb

Fix eelsdb

hyperspy/io_plugins/msa.py

@@ -159,8 +159,8 @@ def parse_msa_string(string, filename=None):
     --------
     file_data_list: list
         The list containts a dictionary that contains the parsed
-        information. It can be used to create a `:class:BaseSignal`
-        using `:func:hyperspy.io.dict2signal`.
+        information. It can be used to create a :py:class:`~.signal.BaseSignal`
+        using :py:func:`~.io.dict2signal`.
 
     """
     if not hasattr(string, "readlines"):
@@ -211,18 +211,26 @@ def parse_msa_string(string, filename=None):
         else:
             clean_par, units = parameter, None
         if clean_par in keywords:
+            type_ = keywords[clean_par]['dtype']
             try:
-                parameters[parameter] = keywords[clean_par]['dtype'](value)
+                parameters[parameter] = type_(value)
             except BaseException:
-                # Normally the offending mispelling is a space in the scientic
-                # notation, e.g. 2.0 E-06, so we try to correct for it
-                try:
-                    parameters[parameter] = keywords[clean_par]['dtype'](
-                        value.replace(' ', ''))
-                except BaseException:
-                    _logger.exception(
-                        "The %s keyword value, %s could not be converted to "
-                        "the right type", parameter, value)
+                error = f"The {parameter} keyword value, {value} could \
+                    not be converted to the right type."
+                if 'e' in value.lower():
+                    # Normally, the offending misspelling is a space in the 
+                    # scientific notation, e.g. 2.0 E-06
+                    try:
+                        parameters[parameter] = type_(value.replace(' ', ''))
+                    except BaseException:  # pragma: no cover
+                        _logger.exception(error)
+                else:
+                    # Some files have two values separated by a space
+                    # https://eelsdb.eu/wp-content/uploads/2017/03/Cu4O3-O-K.msa
+                    try:
+                        parameters[parameter] = type_(value.split(' ')[0])
+                    except BaseException:  # pragma: no cover
+                        _logger.exception(error)
 
             if keywords[clean_par]['mapped_to'] is not None:
                 mapped.set_item(keywords[clean_par]['mapped_to'],
@@ -235,9 +243,10 @@ def parse_msa_string(string, filename=None):
             time = dt.strptime(parameters['TIME'], "%H:%M")
             mapped.set_item('General.time', time.time().isoformat())
         except ValueError as e:
-            _logger.warning('Possible malformed TIME field in msa file. The time information could not be retrieved.: %s' % e)
-    else:
-        _logger.warning('TIME information missing.')
+            _logger.warning(
+                'Possible malformed TIME field in msa file. The time '
+                f'information could not be retrieved.: {e}'
+                )
 
     malformed_date_error = 'Possibly malformed DATE in msa file. The date information could not be retrieved.'
     if "DATE" in parameters and parameters["DATE"]:

hyperspy/misc/eels/eelsdb.py

@@ -20,17 +20,22 @@
 import requests
 import logging
 
+from hyperspy.defaults_parser import preferences
+from hyperspy.docstrings.signal import SHOW_PROGRESSBAR_ARG
+from hyperspy.external.progressbar import progressbar
 from hyperspy.io_plugins.msa import parse_msa_string
 from hyperspy.io import dict2signal
 
+
 _logger = logging.getLogger(__name__)
 
 
 def eelsdb(spectrum_type=None, title=None, author=None, element=None, formula=None,
            edge=None, min_energy=None, max_energy=None, resolution=None,
            min_energy_compare="gt", max_energy_compare="lt",
            resolution_compare="lt", max_n=-1, monochromated=None, order=None,
-           order_direction="ASC", verify_certificate=True):
+           order_direction="ASC", verify_certificate=True,
+           show_progressbar=None):
     r"""Download spectra from the EELS Data Base.
 
     Parameters
@@ -115,6 +120,7 @@ def eelsdb(spectrum_type=None, title=None, author=None, element=None, formula=No
         If True, verify the eelsdb website certificate and raise an error
         if it is invalid. If False, continue querying the database if the certificate
         is invalid. (This is a potential security risk.)
+    %s
 
 
 
@@ -218,16 +224,30 @@ def eelsdb(spectrum_type=None, title=None, author=None, element=None, formula=No
     else:
         params["element[]"] = element
 
+    if show_progressbar is None:
+        show_progressbar = preferences.General.show_progressbar
+
     request = requests.get('http://api.eelsdb.eu/spectra', params=params, verify=verify_certificate)
     spectra = []
     jsons = request.json()
     if "message" in jsons:
         # Invalid query, EELSdb raises error.
         raise IOError(
             "Please report the following error to the HyperSpy developers: "
-            "%s" % jsons["message"])
-    for json_spectrum in jsons:
+            f"{jsons['message']}."
+            )
+
+    for json_spectrum in progressbar(jsons, disable=not show_progressbar):
         download_link = json_spectrum['download_link']
+        if download_link.split('.')[-1].lower() != 'msa':
+            _logger.exception(
+                "The source file is not a msa file, please report this error "
+                "to http://eelsdb.eu/about with the following details:\n"
+                f"Title: {json_spectrum['title']}\nid: {json_spectrum['id']}\n"
+                f"Download link: {download_link}\n"
+                f"Permalink: {json_spectrum['permalink']}"
+                )
+            continue
         msa_string = requests.get(download_link, verify=verify_certificate).text
         try:
             s = dict2signal(parse_msa_string(msa_string)[0])
@@ -246,6 +266,7 @@ def eelsdb(spectrum_type=None, title=None, author=None, element=None, formula=No
                 "Title: %s id: %s.\n"
                 "Please report this error to http://eelsdb.eu/about \n" %
                 (json_spectrum["title"], json_spectrum["id"]))
+
     if not spectra:
         _logger.info(
             "The EELS database does not contain any spectra matching your query"
@@ -261,9 +282,12 @@ def eelsdb(spectrum_type=None, title=None, author=None, element=None, formula=No
             json_md = s.original_metadata.json
             s.metadata.General.title = json_md.title
             if s.metadata.Signal.signal_type == "EELS":
-                if json_md.elements:
+                if json_md.get_item('elements'):
                     try:
-                        s.add_elements(json_md.elements)
+                        # When 'No' is in the list of elements
+                        # https://api.eelsdb.eu/spectra/zero-loss-c-feg-hitachi-disp-0-214-ev/
+                        if json_md.elements[0].lower() != 'no':
+                            s.add_elements(json_md.elements)
                     except ValueError:
                         _logger.exception(
                             "The following spectrum contains invalid chemical "
@@ -294,3 +318,5 @@ def eelsdb(spectrum_type=None, title=None, author=None, element=None, formula=No
                                 json_md.microscope)
 
     return spectra
+
+eelsdb.__doc__ %= SHOW_PROGRESSBAR_ARG

hyperspy/tests/datasets/test_eelsdb.py

@@ -25,6 +25,20 @@
 from hyperspy.misc.eels.eelsdb import eelsdb
 
 
+def _eelsdb(**kwargs):
+    try:
+        ss = eelsdb(**kwargs)
+    except SSLError:
+        warnings.warn(
+            "The https://eelsdb.eu certificate seems to be invalid. "
+            "Consider notifying the issue to the EELSdb webmaster.")
+        ss = eelsdb(verify_certificate=False, **kwargs)
+    except Exception as e:
+        # e.g. failures such as ConnectionError or MaxRetryError
+        pytest.skip(f"Skipping eelsdb test due to {e}")
+    return ss
+
+
 def eelsdb_down():
     try:
         _ = requests.get('http://api.eelsdb.eu', verify=True)
@@ -38,45 +52,22 @@ def eelsdb_down():
 
 @pytest.mark.skipif(eelsdb_down(), reason="Unable to connect to EELSdb")
 def test_eelsdb_eels():
-    try:
-        ss = eelsdb(
-            title="Boron Nitride Multiwall Nanotube",
-            formula="BN",
-            spectrum_type="coreloss",
-            edge="K",
-            min_energy=370,
-            max_energy=1000,
-            min_energy_compare="gt",
-            max_energy_compare="lt",
-            resolution="0.7 eV",
-            resolution_compare="lt",
-            max_n=2,
-            order="spectrumMin",
-            order_direction='DESC',
-            monochromated=False, )
-    except SSLError:
-        warnings.warn(
-            "The https://eelsdb.eu certificate seems to be invalid. "
-            "Consider notifying the issue to the EELSdb webmaster.")
-        ss = eelsdb(
-            title="Boron Nitride Multiwall Nanotube",
-            formula="BN",
-            spectrum_type="coreloss",
-            edge="K",
-            min_energy=370,
-            max_energy=1000,
-            min_energy_compare="gt",
-            max_energy_compare="lt",
-            resolution="0.7 eV",
-            resolution_compare="lt",
-            max_n=2,
-            order="spectrumMin",
-            order_direction='DESC',
-            monochromated=False,
-            verify_certificate=False)
-    except Exception as e:
-        # e.g. failures such as ConnectionError or MaxRetryError
-        pytest.skip(f"Skipping eelsdb test due to {e}")
+    ss = _eelsdb(
+        title="Boron Nitride Multiwall Nanotube",
+        formula="BN",
+        spectrum_type="coreloss",
+        edge="K",
+        min_energy=370,
+        max_energy=1000,
+        min_energy_compare="gt",
+        max_energy_compare="lt",
+        resolution="0.7 eV",
+        resolution_compare="lt",
+        max_n=2,
+        order="spectrumMin",
+        order_direction='DESC',
+        monochromated=False,
+        )
 
     assert len(ss) == 2
     md = ss[0].metadata
@@ -94,16 +85,29 @@ def test_eelsdb_eels():
 
 @pytest.mark.skipif(eelsdb_down(), reason="Unable to connect to EELSdb")
 def test_eelsdb_xas():
-    try:
-        ss = eelsdb(
-            spectrum_type="xrayabs", max_n=1,)
-    except SSLError:
-        ss = eelsdb(
-            spectrum_type="xrayabs", max_n=1, verify_certificate=False)
-    except Exception as e:
-        # e.g. failures such as ConnectionError or MaxRetryError
-        pytest.skip(f"Skipping eelsdb test due to {e}")
-
+    ss = _eelsdb(spectrum_type="xrayabs", max_n=1,)
     assert len(ss) == 1
     md = ss[0].metadata
     assert md.Signal.signal_type == "XAS"
+
+
+@pytest.mark.skipif(eelsdb_down(), reason="Unable to connect to EELSdb")
+def test_eelsdb_corrupted_file():
+    ss = _eelsdb(
+        spectrum_type='coreloss', element='Cu', edge='K', formula='Cu4O3'
+        )
+    assert len(ss) == 1
+    assert ss[0].metadata.General.title == 'O-K edge in Cu4O3'
+
+
+@pytest.mark.skipif(eelsdb_down(), reason="Unable to connect to EELSdb")
+def test_eelsdb_elements_no():
+    title = "Zero-loss c-FEG Hitachi Disp 0.214 eV"
+    ss = _eelsdb(author='Luc Lajaunie', title=title)
+    assert len(ss) == 1
+    assert ss[0].metadata.General.title == title
+
+
+@pytest.mark.skipif(eelsdb_down(), reason="Unable to connect to EELSdb")
+def test_eelsdb_txt_file():
+    _ = _eelsdb(title="Porous cobalt coating with He-filled nanopores")

upcoming_changes/2916.bugfix.rst

@@ -0,0 +1 @@
+Fix bug with importing some spectra from eelsdb and add progress bar