major refactoring of JPK metadata reader

 - ref: JPK calibration files are now not ignored anymore
 - ref: allow to specify custom metadata for JPK files
 - ref: replacef ReadJPKMetaKeyError with MissingMetaDataError
 - ref: support `return_modality` in `detect` methods of recipes
 - ref: cleanup of JPK reader class

CHANGELOG

@@ -1,3 +1,9 @@
+0.18.0
+ - ref: JPK calibration files are now not ignored anymore
+ - ref: allow to specify custom metadata for JPK files
+ - ref: replacef ReadJPKMetaKeyError with MissingMetaDataError
+ - ref: support `return_modality` in `detect` methods of recipes
+ - ref: cleanup of JPK reader class
 0.17.1
  - maintenance release
 0.17.0

afmformats/formats/__init__.py

@@ -134,8 +134,8 @@ def get_modality(self, path):
         if len(self.modalities) == 1:
             modality = self.modalities[0]
         else:
-            metadata = self.loader(path)[0]["metadata"]
-            modality = metadata["imaging mode"]
+            fdetect = self.recipe["detect"]
+            _, modality = fdetect(path, return_modality=True)
         return modality
 
 
@@ -254,7 +254,8 @@ def load_data(path, meta_override=None, modality=None,
             afm_data_class = data_classes_by_modality[modality]
         else:
             afm_data_class = default_data_classes_by_modality[modality]
-        for dd in loader(path, callback=callback,
+        for dd in loader(path,
+                         callback=callback,
                          meta_override=meta_override):
             dd["metadata"]["format"] = "{} ({})".format(cur_recipe["maker"],
                                                         cur_recipe["descr"])

afmformats/formats/fmt_jpk/__init__.py

@@ -1,16 +1,16 @@
 import numpy as np
 
+from ...errors import MissingMetaDataError
 from ...lazy_loader import LazyData
 from ...meta import LazyMetaValue
 
 from .jpk_reader import JPKReader
-from .jpk_meta import ReadJPKMetaKeyError
 
 
 __all__ = ["load_jpk"]
 
 
-def detect(path):
+def detect(path, return_modality=False):
     """Check whether a file is a valid JPK data file
 
     """
@@ -19,11 +19,16 @@ def detect(path):
     jpkr = JPKReader(path)
     try:
         jpkr.get_metadata(index=0)
-    except ReadJPKMetaKeyError:
+    except MissingMetaDataError:
+        valid = True
+    except BaseException:
         valid = False
     else:
         valid = True
-    return valid
+    if return_modality:
+        return valid, jpkr.get_imaging_mode()
+    else:
+        return valid
 
 
 def load_jpk(path, callback=None, meta_override=None):
@@ -47,14 +52,10 @@ def load_jpk(path, callback=None, meta_override=None):
     """
     if meta_override is None:
         meta_override = {}
-    else:
-        # just make sure nobody expects a different result for the forces
-        for key in ["sensitivity", "spring constant"]:
-            if key in meta_override:
-                raise NotImplementedError(
-                    f"Setting metadata such as '{key}' is not implemented!")
 
     jpkr = JPKReader(path)
+    jpkr.set_metadata(meta_override)
+
     dataset = []
     # iterate over all datasets and add them
     for index in range(len(jpkr)):
@@ -69,7 +70,6 @@ def load_jpk(path, callback=None, meta_override=None):
         metadata["z range"] = LazyMetaValue(
             lambda data: np.ptp(data["height (piezo)"]),
             lazy_data)
-        metadata.update(meta_override)
         dataset.append({"data": lazy_data,
                         "metadata": metadata,
                         })

afmformats/formats/fmt_jpk/jpk_meta.py

@@ -2,19 +2,12 @@
 import json
 from pkg_resources import resource_filename
 
-from ...errors import FileFormatMetaDataError
 
-
-__all__ = ["ReadJPKMetaKeyError",
-           "get_primary_meta_recipe",
+__all__ = ["get_primary_meta_recipe",
            "get_secondary_meta_recipe",
            ]
 
 
-class ReadJPKMetaKeyError(FileFormatMetaDataError):
-    pass
-
-
 @functools.lru_cache()
 def get_primary_meta_recipe():
     with open(resource_filename("afmformats.formats.fmt_jpk",

afmformats/formats/fmt_jpk/jpk_reader.py

@@ -6,6 +6,7 @@
 import jprops
 import numpy as np
 
+from ...errors import MissingMetaDataError
 from ... import meta
 
 from . import jpk_data, jpk_meta
@@ -53,6 +54,7 @@ def get(zip_path):
 class JPKReader(object):
     def __init__(self, path):
         self.path = path
+        self._user_metadata = {}
 
     @functools.lru_cache()
     def __len__(self):
@@ -115,7 +117,7 @@ def _properties_shared(self):
 
     @functools.lru_cache()
     def _get_index_segment_properties(self, index, segment):
-        """Return properties fro a specific index and segment
+        """Return properties from a specific index and segment
 
         Parameters
         ----------
@@ -328,36 +330,30 @@ def get_metadata(self, index, segment=None):
                     md["point count"] += mdap.pop("point count")
                 md.update(mdap)
             return md
-        prop = self._get_index_segment_properties(index=index, segment=segment)
+
         # 1. Populate with primary metadata
-        md = meta.MetaData()
-        recipe = jpk_meta.get_primary_meta_recipe()
-        for key in recipe:
-            for vari in recipe[key]:
-                if vari in prop:
-                    md[key] = prop[vari]
-                    break
-        for mkey in ["spring constant",
-                     "sensitivity",
-                     ]:
-            if mkey not in md:
-                msg = "Missing meta data: '{}'".format(mkey)
-                raise jpk_meta.ReadJPKMetaKeyError(msg)
+        md = self.get_metadata_jpk_primary(index=index, segment=segment)
+
+        # Make sure that spring constant and sensitivity are in the metadata,
+        # otherwise fail. We also need those two for computations further
+        # below.
+        md.update(self._user_metadata)
+
+        keys_required = ["spring constant", "sensitivity"]
+        keys_missing = [k for k in keys_required if k not in md]
+
+        if keys_missing:
+            raise MissingMetaDataError(keys_missing,
+                                       f"Missing meta data: '{keys_missing}'"
+                                       )
 
         md["software"] = "JPK"
 
         md["enum"] = int(self.get_index_numbers()[index])
         md["path"] = self.path
 
         # 2. Populate with secondary metadata
-        recipe_2 = jpk_meta.get_secondary_meta_recipe()
-        md_im = {}
-
-        for key in recipe_2:
-            for vari in recipe_2[key]:
-                if vari in prop:
-                    md_im[key] = prop[vari]
-                    break
+        md_im = self.get_metadata_jpk_secondary(index=index, segment=segment)
 
         curve_type = md_im["curve type"]
         curve_conv = {"extend": "approach",
@@ -372,33 +368,7 @@ def get_metadata(self, index, segment=None):
             md[f"speed {curseg}"] = zrange / md_im["segment duration"]
         md[f"duration {curseg}"] = md_im["segment duration"]
 
-        num_segments = len(self.get_index_segment_numbers(0))
-        if num_segments == 2:
-            md["imaging mode"] = "force-distance"
-        elif num_segments == 3:
-            if segment == 1:
-                # For creep-compliance, we extract the info from segment 1.
-                if md_im["curve type"] != "pause":
-                    raise ValueError(
-                        f"Segment {segment} must be of type 'pause'!")
-                pause_type = md_im["segment pause type"]
-                if pause_type == "constant-force-pause":
-                    md["imaging mode"] = "creep-compliance"
-                else:
-                    raise ValueError(f"Unexprected pause type '{pause_type}'!")
-        elif num_segments == 4:
-            if segment == 2:
-                # For stress-relaxation, we extract the info from segment 2.
-                if md_im["curve type"] != "pause":
-                    raise ValueError(
-                        f"Segment {segment} must be of type 'pause'!")
-                pause_type = md_im["segment pause type"]
-                if pause_type == "constant-height-pause":
-                    md["imaging mode"] = "stress-relaxation"
-                else:
-                    raise ValueError(f"Unexprected pause type '{pause_type}'!")
-        else:
-            raise ValueError(f"Unexpected number of segments: {num_segments}!")
+        md["imaging mode"] = self.get_imaging_mode()
 
         md["curve id"] = "{}:{:g}".format(md["session id"],
                                           md_im["position index"])
@@ -420,4 +390,70 @@ def get_metadata(self, index, segment=None):
         for ik in integer_keys:
             if ik in md:
                 md[ik] = int(round(md[ik]))
+
+        # Update any remaining metadata from the user. This is not spotless
+        # (it might have made more sense to use `dict.setdefault`
+        # more often).
+        md.update(self._user_metadata)
+        return md
+
+    def get_metadata_jpk_primary(self, index, segment):
+        prop = self._get_index_segment_properties(index=index, segment=segment)
+        md = meta.MetaData()
+        recipe = jpk_meta.get_primary_meta_recipe()
+        for key in recipe:
+            for vari in recipe[key]:
+                if vari in prop:
+                    md[key] = prop[vari]
+                    break
         return md
+
+    def get_metadata_jpk_secondary(self, index, segment):
+        prop = self._get_index_segment_properties(index=index, segment=segment)
+        recipe_2 = jpk_meta.get_secondary_meta_recipe()
+        md_im = {}
+
+        for key in recipe_2:
+            for vari in recipe_2[key]:
+                if vari in prop:
+                    md_im[key] = prop[vari]
+                    break
+        return md_im
+
+    @functools.lru_cache()
+    def get_imaging_mode(self):
+        num_segments = len(self.get_index_segment_numbers(0))
+        if num_segments == 2:
+            imaging_mode = "force-distance"
+        elif num_segments == 3:
+            md_im = self.get_metadata_jpk_secondary(index=0, segment=1)
+            # For creep-compliance, we extract the info from segment 1.
+            if md_im["curve type"] != "pause":
+                raise ValueError("Segment 1 must be of type 'pause'!")
+            pause_type = md_im["segment pause type"]
+            if pause_type == "constant-force-pause":
+                imaging_mode = "creep-compliance"
+            else:
+                raise ValueError(f"Unexprected pause type '{pause_type}'!")
+        elif num_segments == 4:
+            md_im = self.get_metadata_jpk_secondary(index=0, segment=2)
+            # For stress-relaxation, we extract the info from segment 2.
+            if md_im["curve type"] != "pause":
+                raise ValueError("Segment 2 must be of type 'pause'!")
+            pause_type = md_im["segment pause type"]
+            if pause_type == "constant-height-pause":
+                imaging_mode = "stress-relaxation"
+            else:
+                raise ValueError(f"Unexprected pause type '{pause_type}'!")
+        else:
+            raise ValueError(f"Unexpected number of segments: {num_segments}!")
+        return imaging_mode
+
+    def set_metadata(self, metadata):
+        """Override internal metadata
+
+        This has a direct effect on :func:`.get_metadata`.
+        """
+        self._user_metadata.clear()
+        self._user_metadata.update(metadata)
+        self.get_metadata.cache_clear()

tests/test_afm_recipe.py

@@ -71,8 +71,9 @@ def test_bad_recipe_suffix_invalid():
 @pytest.mark.parametrize("name, is_valid",
     [("fmt-jpk-fd_spot3-0192.jpk-force", True),  # noqa: E128
      ("fmt-jpk-fd_map2x2_extracted.jpk-force-map", True),
+     # Since version 0.18.0, afmformats supports opening calibration curves
      ("fmt-jpk-cl_calibration_force-save-2015.02.04-11.25.21.294.jpk-force",
-      False),
+      True),
      ])
 def test_find_data(name, is_valid):
     file = data_path / name
@@ -92,7 +93,8 @@ def test_find_data_recursively():
         "fmt-jpk-cl_calibration_force-save-2015.02.04-11.25.21.294.jpk-force",
         td3)
     file_list = afmformats.find_data(td)
-    assert len(file_list) == 1
+    # Since version 0.18.0, afmformats supports opening calibration curves
+    assert len(file_list) == 2
     assert file_list[0].samefile(td2 / "fmt-jpk-fd_spot3-0192.jpk-force")
 
 

tests/test_fmt.py

@@ -3,12 +3,19 @@
 import pytest
 
 import afmformats
+import afmformats.formats
 import afmformats.errors
 
 
 data_path = pathlib.Path(__file__).parent / "data"
 
 
+@pytest.mark.parametrize("path", data_path.glob("fmt-*-fd_*"))
+def test_load_force_distance_modality(path):
+    recipe = afmformats.formats.get_recipe(path)
+    assert recipe.get_modality(path) == "force-distance"
+
+
 @pytest.mark.parametrize("path", data_path.glob("fmt-*-fd_*"))
 def test_load_force_distance_with_callback(path):
     """Make sure that the callback function is properly implemented"""
@@ -27,6 +34,12 @@ def callback(value):
     assert calls[-1] == 1
 
 
+@pytest.mark.parametrize("path", data_path.glob("fmt-*-cc_*"))
+def test_load_creep_compliance_modality(path):
+    recipe = afmformats.formats.get_recipe(path)
+    assert recipe.get_modality(path) == "creep-compliance"
+
+
 @pytest.mark.parametrize("path", data_path.glob("fmt-*-cc_*"))
 def test_load_creep_compliance_with_callback(path):
     """Make sure that the callback function is properly implemented"""

tests/test_fmt_jpk_basic.py

@@ -41,18 +41,20 @@ def test_creep_compliance2():
                        atol=0, rtol=1e-12)
 
 
-@pytest.mark.parametrize("name, is_valid",
+@pytest.mark.parametrize("name, exp_valid",
     [("fmt-jpk-fd_spot3-0192.jpk-force", True),  # noqa: E128
      ("fmt-jpk-fd_map2x2_extracted.jpk-force-map", True),
+     # Since 0.18.0 afmformats supports opening calibration files
      ("fmt-jpk-cl_calibration_force-save-2015.02.04-11.25.21.294.jpk-force",
-      False),
+      True),
      ])
-def test_detect_jpk(name, is_valid):
+def test_detect_jpk(name, exp_valid):
     jpkfile = data_path / name
-    if is_valid:
-        afmlist = afmformats.load_data(path=jpkfile)
-        assert afmlist
-    else:
+    recipe = afmformats.formats.get_recipe(path=jpkfile)
+    act_valid = recipe.detect(jpkfile)
+    assert exp_valid == act_valid
+
+    if not exp_valid:
         with pytest.raises(afmformats.errors.FileFormatNotSupportedError):
             afmformats.load_data(path=jpkfile)
 
@@ -94,11 +96,3 @@ def test_load_jpk_piezo():
     afmlist = afmformats.load_data(path=jpkfile)
     ds = afmlist[0]
     assert np.allclose(ds["height (piezo)"][0], 2.878322343068329e-05)
-
-
-if __name__ == "__main__":
-    # Run all tests
-    _loc = locals()
-    for _key in list(_loc.keys()):
-        if _key.startswith("test_") and hasattr(_loc[_key], "__call__"):
-            _loc[_key]()