diff --git a/CHANGELOG b/CHANGELOG index 687e3be..d675f88 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +0.18.0 + - ref: JPK calibration files are now not ignored anymore + - ref: allow to specify custom metadata for JPK files + - ref: replacef ReadJPKMetaKeyError with MissingMetaDataError + - ref: support `return_modality` in `detect` methods of recipes + - ref: cleanup of JPK reader class 0.17.1 - maintenance release 0.17.0 diff --git a/afmformats/formats/__init__.py b/afmformats/formats/__init__.py index ca3d776..d03714d 100644 --- a/afmformats/formats/__init__.py +++ b/afmformats/formats/__init__.py @@ -134,8 +134,8 @@ def get_modality(self, path): if len(self.modalities) == 1: modality = self.modalities[0] else: - metadata = self.loader(path)[0]["metadata"] - modality = metadata["imaging mode"] + fdetect = self.recipe["detect"] + _, modality = fdetect(path, return_modality=True) return modality @@ -254,7 +254,8 @@ def load_data(path, meta_override=None, modality=None, afm_data_class = data_classes_by_modality[modality] else: afm_data_class = default_data_classes_by_modality[modality] - for dd in loader(path, callback=callback, + for dd in loader(path, + callback=callback, meta_override=meta_override): dd["metadata"]["format"] = "{} ({})".format(cur_recipe["maker"], cur_recipe["descr"]) diff --git a/afmformats/formats/fmt_jpk/__init__.py b/afmformats/formats/fmt_jpk/__init__.py index bb12d6a..b0da6ec 100644 --- a/afmformats/formats/fmt_jpk/__init__.py +++ b/afmformats/formats/fmt_jpk/__init__.py @@ -1,16 +1,16 @@ import numpy as np +from ...errors import MissingMetaDataError from ...lazy_loader import LazyData from ...meta import LazyMetaValue from .jpk_reader import JPKReader -from .jpk_meta import ReadJPKMetaKeyError __all__ = ["load_jpk"] -def detect(path): +def detect(path, return_modality=False): """Check whether a file is a valid JPK data file """ @@ -19,11 +19,16 @@ def detect(path): jpkr = JPKReader(path) try: jpkr.get_metadata(index=0) - except ReadJPKMetaKeyError: + except MissingMetaDataError: + valid = True + except BaseException: valid = False else: valid = True - return valid + if return_modality: + return valid, jpkr.get_imaging_mode() + else: + return valid def load_jpk(path, callback=None, meta_override=None): @@ -47,14 +52,10 @@ def load_jpk(path, callback=None, meta_override=None): """ if meta_override is None: meta_override = {} - else: - # just make sure nobody expects a different result for the forces - for key in ["sensitivity", "spring constant"]: - if key in meta_override: - raise NotImplementedError( - f"Setting metadata such as '{key}' is not implemented!") jpkr = JPKReader(path) + jpkr.set_metadata(meta_override) + dataset = [] # iterate over all datasets and add them for index in range(len(jpkr)): @@ -69,7 +70,6 @@ def load_jpk(path, callback=None, meta_override=None): metadata["z range"] = LazyMetaValue( lambda data: np.ptp(data["height (piezo)"]), lazy_data) - metadata.update(meta_override) dataset.append({"data": lazy_data, "metadata": metadata, }) diff --git a/afmformats/formats/fmt_jpk/jpk_meta.py b/afmformats/formats/fmt_jpk/jpk_meta.py index 9b54ce8..3065d21 100644 --- a/afmformats/formats/fmt_jpk/jpk_meta.py +++ b/afmformats/formats/fmt_jpk/jpk_meta.py @@ -2,19 +2,12 @@ import json from pkg_resources import resource_filename -from ...errors import FileFormatMetaDataError - -__all__ = ["ReadJPKMetaKeyError", - "get_primary_meta_recipe", +__all__ = ["get_primary_meta_recipe", "get_secondary_meta_recipe", ] -class ReadJPKMetaKeyError(FileFormatMetaDataError): - pass - - @functools.lru_cache() def get_primary_meta_recipe(): with open(resource_filename("afmformats.formats.fmt_jpk", diff --git a/afmformats/formats/fmt_jpk/jpk_reader.py b/afmformats/formats/fmt_jpk/jpk_reader.py index ac277e7..901f074 100644 --- a/afmformats/formats/fmt_jpk/jpk_reader.py +++ b/afmformats/formats/fmt_jpk/jpk_reader.py @@ -6,6 +6,7 @@ import jprops import numpy as np +from ...errors import MissingMetaDataError from ... import meta from . import jpk_data, jpk_meta @@ -53,6 +54,7 @@ def get(zip_path): class JPKReader(object): def __init__(self, path): self.path = path + self._user_metadata = {} @functools.lru_cache() def __len__(self): @@ -115,7 +117,7 @@ def _properties_shared(self): @functools.lru_cache() def _get_index_segment_properties(self, index, segment): - """Return properties fro a specific index and segment + """Return properties from a specific index and segment Parameters ---------- @@ -328,21 +330,22 @@ def get_metadata(self, index, segment=None): md["point count"] += mdap.pop("point count") md.update(mdap) return md - prop = self._get_index_segment_properties(index=index, segment=segment) + # 1. Populate with primary metadata - md = meta.MetaData() - recipe = jpk_meta.get_primary_meta_recipe() - for key in recipe: - for vari in recipe[key]: - if vari in prop: - md[key] = prop[vari] - break - for mkey in ["spring constant", - "sensitivity", - ]: - if mkey not in md: - msg = "Missing meta data: '{}'".format(mkey) - raise jpk_meta.ReadJPKMetaKeyError(msg) + md = self.get_metadata_jpk_primary(index=index, segment=segment) + + # Make sure that spring constant and sensitivity are in the metadata, + # otherwise fail. We also need those two for computations further + # below. + md.update(self._user_metadata) + + keys_required = ["spring constant", "sensitivity"] + keys_missing = [k for k in keys_required if k not in md] + + if keys_missing: + raise MissingMetaDataError(keys_missing, + f"Missing meta data: '{keys_missing}'" + ) md["software"] = "JPK" @@ -350,14 +353,7 @@ def get_metadata(self, index, segment=None): md["path"] = self.path # 2. Populate with secondary metadata - recipe_2 = jpk_meta.get_secondary_meta_recipe() - md_im = {} - - for key in recipe_2: - for vari in recipe_2[key]: - if vari in prop: - md_im[key] = prop[vari] - break + md_im = self.get_metadata_jpk_secondary(index=index, segment=segment) curve_type = md_im["curve type"] curve_conv = {"extend": "approach", @@ -372,33 +368,7 @@ def get_metadata(self, index, segment=None): md[f"speed {curseg}"] = zrange / md_im["segment duration"] md[f"duration {curseg}"] = md_im["segment duration"] - num_segments = len(self.get_index_segment_numbers(0)) - if num_segments == 2: - md["imaging mode"] = "force-distance" - elif num_segments == 3: - if segment == 1: - # For creep-compliance, we extract the info from segment 1. - if md_im["curve type"] != "pause": - raise ValueError( - f"Segment {segment} must be of type 'pause'!") - pause_type = md_im["segment pause type"] - if pause_type == "constant-force-pause": - md["imaging mode"] = "creep-compliance" - else: - raise ValueError(f"Unexprected pause type '{pause_type}'!") - elif num_segments == 4: - if segment == 2: - # For stress-relaxation, we extract the info from segment 2. - if md_im["curve type"] != "pause": - raise ValueError( - f"Segment {segment} must be of type 'pause'!") - pause_type = md_im["segment pause type"] - if pause_type == "constant-height-pause": - md["imaging mode"] = "stress-relaxation" - else: - raise ValueError(f"Unexprected pause type '{pause_type}'!") - else: - raise ValueError(f"Unexpected number of segments: {num_segments}!") + md["imaging mode"] = self.get_imaging_mode() md["curve id"] = "{}:{:g}".format(md["session id"], md_im["position index"]) @@ -420,4 +390,70 @@ def get_metadata(self, index, segment=None): for ik in integer_keys: if ik in md: md[ik] = int(round(md[ik])) + + # Update any remaining metadata from the user. This is not spotless + # (it might have made more sense to use `dict.setdefault` + # more often). + md.update(self._user_metadata) + return md + + def get_metadata_jpk_primary(self, index, segment): + prop = self._get_index_segment_properties(index=index, segment=segment) + md = meta.MetaData() + recipe = jpk_meta.get_primary_meta_recipe() + for key in recipe: + for vari in recipe[key]: + if vari in prop: + md[key] = prop[vari] + break return md + + def get_metadata_jpk_secondary(self, index, segment): + prop = self._get_index_segment_properties(index=index, segment=segment) + recipe_2 = jpk_meta.get_secondary_meta_recipe() + md_im = {} + + for key in recipe_2: + for vari in recipe_2[key]: + if vari in prop: + md_im[key] = prop[vari] + break + return md_im + + @functools.lru_cache() + def get_imaging_mode(self): + num_segments = len(self.get_index_segment_numbers(0)) + if num_segments == 2: + imaging_mode = "force-distance" + elif num_segments == 3: + md_im = self.get_metadata_jpk_secondary(index=0, segment=1) + # For creep-compliance, we extract the info from segment 1. + if md_im["curve type"] != "pause": + raise ValueError("Segment 1 must be of type 'pause'!") + pause_type = md_im["segment pause type"] + if pause_type == "constant-force-pause": + imaging_mode = "creep-compliance" + else: + raise ValueError(f"Unexprected pause type '{pause_type}'!") + elif num_segments == 4: + md_im = self.get_metadata_jpk_secondary(index=0, segment=2) + # For stress-relaxation, we extract the info from segment 2. + if md_im["curve type"] != "pause": + raise ValueError("Segment 2 must be of type 'pause'!") + pause_type = md_im["segment pause type"] + if pause_type == "constant-height-pause": + imaging_mode = "stress-relaxation" + else: + raise ValueError(f"Unexprected pause type '{pause_type}'!") + else: + raise ValueError(f"Unexpected number of segments: {num_segments}!") + return imaging_mode + + def set_metadata(self, metadata): + """Override internal metadata + + This has a direct effect on :func:`.get_metadata`. + """ + self._user_metadata.clear() + self._user_metadata.update(metadata) + self.get_metadata.cache_clear() diff --git a/tests/test_afm_recipe.py b/tests/test_afm_recipe.py index e7ee091..b25c073 100644 --- a/tests/test_afm_recipe.py +++ b/tests/test_afm_recipe.py @@ -71,8 +71,9 @@ def test_bad_recipe_suffix_invalid(): @pytest.mark.parametrize("name, is_valid", [("fmt-jpk-fd_spot3-0192.jpk-force", True), # noqa: E128 ("fmt-jpk-fd_map2x2_extracted.jpk-force-map", True), + # Since version 0.18.0, afmformats supports opening calibration curves ("fmt-jpk-cl_calibration_force-save-2015.02.04-11.25.21.294.jpk-force", - False), + True), ]) def test_find_data(name, is_valid): file = data_path / name @@ -92,7 +93,8 @@ def test_find_data_recursively(): "fmt-jpk-cl_calibration_force-save-2015.02.04-11.25.21.294.jpk-force", td3) file_list = afmformats.find_data(td) - assert len(file_list) == 1 + # Since version 0.18.0, afmformats supports opening calibration curves + assert len(file_list) == 2 assert file_list[0].samefile(td2 / "fmt-jpk-fd_spot3-0192.jpk-force") diff --git a/tests/test_fmt.py b/tests/test_fmt.py index 20356ae..13595d3 100644 --- a/tests/test_fmt.py +++ b/tests/test_fmt.py @@ -3,12 +3,19 @@ import pytest import afmformats +import afmformats.formats import afmformats.errors data_path = pathlib.Path(__file__).parent / "data" +@pytest.mark.parametrize("path", data_path.glob("fmt-*-fd_*")) +def test_load_force_distance_modality(path): + recipe = afmformats.formats.get_recipe(path) + assert recipe.get_modality(path) == "force-distance" + + @pytest.mark.parametrize("path", data_path.glob("fmt-*-fd_*")) def test_load_force_distance_with_callback(path): """Make sure that the callback function is properly implemented""" @@ -27,6 +34,12 @@ def callback(value): assert calls[-1] == 1 +@pytest.mark.parametrize("path", data_path.glob("fmt-*-cc_*")) +def test_load_creep_compliance_modality(path): + recipe = afmformats.formats.get_recipe(path) + assert recipe.get_modality(path) == "creep-compliance" + + @pytest.mark.parametrize("path", data_path.glob("fmt-*-cc_*")) def test_load_creep_compliance_with_callback(path): """Make sure that the callback function is properly implemented""" diff --git a/tests/test_fmt_jpk_basic.py b/tests/test_fmt_jpk_basic.py index bd769ef..9d587e5 100644 --- a/tests/test_fmt_jpk_basic.py +++ b/tests/test_fmt_jpk_basic.py @@ -41,18 +41,20 @@ def test_creep_compliance2(): atol=0, rtol=1e-12) -@pytest.mark.parametrize("name, is_valid", +@pytest.mark.parametrize("name, exp_valid", [("fmt-jpk-fd_spot3-0192.jpk-force", True), # noqa: E128 ("fmt-jpk-fd_map2x2_extracted.jpk-force-map", True), + # Since 0.18.0 afmformats supports opening calibration files ("fmt-jpk-cl_calibration_force-save-2015.02.04-11.25.21.294.jpk-force", - False), + True), ]) -def test_detect_jpk(name, is_valid): +def test_detect_jpk(name, exp_valid): jpkfile = data_path / name - if is_valid: - afmlist = afmformats.load_data(path=jpkfile) - assert afmlist - else: + recipe = afmformats.formats.get_recipe(path=jpkfile) + act_valid = recipe.detect(jpkfile) + assert exp_valid == act_valid + + if not exp_valid: with pytest.raises(afmformats.errors.FileFormatNotSupportedError): afmformats.load_data(path=jpkfile) @@ -94,11 +96,3 @@ def test_load_jpk_piezo(): afmlist = afmformats.load_data(path=jpkfile) ds = afmlist[0] assert np.allclose(ds["height (piezo)"][0], 2.878322343068329e-05) - - -if __name__ == "__main__": - # Run all tests - _loc = locals() - for _key in list(_loc.keys()): - if _key.startswith("test_") and hasattr(_loc[_key], "__call__"): - _loc[_key]() diff --git a/tests/test_fmt_jpk_single.py b/tests/test_fmt_jpk_single.py index 1d92999..1b7a977 100644 --- a/tests/test_fmt_jpk_single.py +++ b/tests/test_fmt_jpk_single.py @@ -2,8 +2,11 @@ import pathlib import numpy as np +import pytest -from afmformats.formats.fmt_jpk import jpk_data, jpk_meta, load_jpk + +import afmformats.errors +from afmformats.formats.fmt_jpk import jpk_data, load_jpk from afmformats.formats.fmt_jpk.jpk_reader import ArchiveCache, JPKReader @@ -27,12 +30,10 @@ def test_open_jpk_simple(): def test_open_jpk_calibration(): cf = data_path / \ "fmt-jpk-cl_calibration_force-save-2015.02.04-11.25.21.294.jpk-force" - try: + + with pytest.raises(afmformats.errors.MissingMetaDataError, + match="sensitivity"): load_jpk(cf) - except jpk_meta.ReadJPKMetaKeyError: - pass - else: - assert False, "no spring constant should raise error" def test_open_jpk_conversion(): @@ -121,6 +122,34 @@ def test_meta(): assert md["spring constant"] == 0.043493666407368466 +def test_meta_missing(): + jpkfile = data_path / "fmt-jpk-fd_single-modified_2023.jpk-force" + + with pytest.raises(afmformats.errors.MissingMetaDataError, + match="spring constant"): + load_jpk(jpkfile) + + data_list = load_jpk(jpkfile, + meta_override={"spring constant": 12}) + meta = data_list[0]["metadata"] + assert meta["spring constant"] == 12 + + +def test_meta_override_multiple_times(): + jpkfile = data_path / "fmt-jpk-fd_single-modified_2023.jpk-force" + jpkr = JPKReader(jpkfile) + + with pytest.raises(afmformats.errors.MissingMetaDataError, + match="spring constant"): + jpkr.get_metadata(index=0) + + jpkr.set_metadata({"spring constant": 10}) + assert jpkr.get_metadata(index=0)["spring constant"] == 10 + + jpkr.set_metadata({"spring constant": 12}) + assert jpkr.get_metadata(index=0)["spring constant"] == 12 + + def test_load_jpk(): jpkfile = data_path / "fmt-jpk-fd_spot3-0192.jpk-force" jpkr = JPKReader(jpkfile) @@ -128,11 +157,3 @@ def test_load_jpk(): assert md["imaging mode"] == "force-distance" assert len(jpkr) == 1, "Only one measurement" assert len(jpkr.get_index_segment_numbers(0)) == 2, "approach and retract" - - -if __name__ == "__main__": - # Run all tests - _loc = locals() - for _key in list(_loc.keys()): - if _key.startswith("test_") and hasattr(_loc[_key], "__call__"): - _loc[_key]()