diff --git a/doc/user_guide/supported_formats/renishaw.rst b/doc/user_guide/supported_formats/renishaw.rst index a062e7701..12ce16249 100644 --- a/doc/user_guide/supported_formats/renishaw.rst +++ b/doc/user_guide/supported_formats/renishaw.rst @@ -5,6 +5,10 @@ Renishaw Reader for spectroscopy data saved using Renishaw's WiRE software. Currently, RosettaSciIO can only read the ``.wdf`` format from Renishaw. +When reading spectral images, the white light image will be returned along the +spectral images in the list of dictionaries. The position of the mapped area +is returned in the metadata dictionary of the white light image and this will +be displayed when plotting the image with HyperSpy. If `LumiSpy `_ is installed, ``Luminescence`` will be used as the ``signal_type``. diff --git a/rsciio/_docstrings.py b/rsciio/_docstrings.py index f173c3478..97097c7f3 100644 --- a/rsciio/_docstrings.py +++ b/rsciio/_docstrings.py @@ -132,4 +132,6 @@ containing the full axes vector - 'metadata' – dictionary containing the parsed metadata - 'original_metadata' – dictionary containing the full metadata tree from the input file + + When the file contains several datasets, each dataset will be loaded as separate dictionary. """ diff --git a/rsciio/image/_api.py b/rsciio/image/_api.py index 4ea52f91f..4401d87e2 100644 --- a/rsciio/image/_api.py +++ b/rsciio/image/_api.py @@ -20,6 +20,7 @@ import logging import imageio.v3 as iio +from PIL import Image import numpy as np from rsciio._docstrings import ( @@ -28,6 +29,7 @@ RETURNS_DOC, SIGNAL_DOC, ) +from rsciio.utils.image import _parse_axes_from_metadata, _parse_exif_tags from rsciio.utils.tools import _UREG @@ -230,13 +232,22 @@ def file_reader(filename, lazy=False, **kwds): dc = from_delayed(val, shape=dc.shape, dtype=dc.dtype) else: dc = _read_data(filename, **kwds) + + om = {} + + im = Image.open(filename) + om["exif_tags"] = _parse_exif_tags(im) + axes = _parse_axes_from_metadata(om["exif_tags"], dc.shape) + return [ { "data": dc, + "axes": axes, "metadata": { "General": {"original_filename": os.path.split(filename)[1]}, "Signal": {"signal_type": ""}, }, + "original_metadata": om, } ] diff --git a/rsciio/renishaw/_api.py b/rsciio/renishaw/_api.py index 18753ac7f..a1ef35fcb 100644 --- a/rsciio/renishaw/_api.py +++ b/rsciio/renishaw/_api.py @@ -66,21 +66,23 @@ from enum import IntEnum, Enum, EnumMeta from io import BytesIO from pathlib import Path +import os import numpy as np from numpy.polynomial.polynomial import polyfit from rsciio._docstrings import FILENAME_DOC, LAZY_DOC, RETURNS_DOC +from rsciio.utils import rgb_tools + _logger = logging.getLogger(__name__) -## PIL alternative: imageio.v3.immeta extracts exif as binary -## but then this binary string needs to be parsed + try: from PIL import Image except ImportError: PIL_installed = False - _logger.warning("Pillow not installed. Cannot load whitelight image into metadata") + _logger.warning("Pillow not installed. Cannot load whitelight image.") else: PIL_installed = True @@ -323,21 +325,6 @@ class DataType(IntEnum, metaclass=DefaultEnumMeta): ) -# for wthl image -class ExifTags(IntEnum, metaclass=DefaultEnumMeta): - # Standard EXIF TAGS - ImageDescription = 0x10E # 270 - Make = 0x10F # 271 - ExifOffset = 0x8769 # 34665 - FocalPlaneXResolution = 0xA20E # 41486 - FocalPlaneYResolution = 0xA20F # 41487 - FocalPlaneResolutionUnit = 0xA210 # 41488 - # Customized EXIF TAGS from Renishaw - FocalPlaneXYOrigins = 0xFEA0 # 65184 - FieldOfViewXY = 0xFEA1 # 65185 - Unknown = 0xFEA2 # 65186 - - class WDFReader(object): """Reader for Renishaw(TM) WiRE Raman spectroscopy files (.wdf format) @@ -469,7 +456,6 @@ def read_file(self, filesize): self._parse_MAP("MAP_0") self._parse_MAP("MAP_1") self._parse_TEXT() - self._parse_WHTL() ## parse blocks with axes information signal_dict = self._parse_XLST() @@ -1025,7 +1011,7 @@ def _set_nav_via_WMAP(self, wmap_dict, units): if flag == MapType.xyline.name: result = self._set_wmap_nav_linexy(result["X"], result["Y"]) elif flag == DefaultEnum.Unknown.name: - _logger.warning(f"Unknown flag ({wmap_dict['flag']}) for WMAP mapping.") + _logger.info(f"Unknown flag ({wmap_dict['flag']}) for WMAP mapping.") return result def _set_wmap_nav_linexy(self, x_axis, y_axis): @@ -1062,13 +1048,32 @@ def _set_nav_via_ORGN(self, orgn_data): ) for axis in orgn_data.keys(): del nav_dict[axis]["annotation"] + data = nav_dict[axis].pop("data") nav_dict[axis]["navigate"] = True - data = np.unique(nav_dict[axis].pop("data")) nav_dict[axis]["size"] = data.size - nav_dict[axis]["offset"] = data[0] - ## time axis in test data is not perfectly uniform, but X,Y,Z are - nav_dict[axis]["scale"] = np.mean(np.diff(data)) nav_dict[axis]["name"] = axis + scale_mean = np.mean(np.diff(data)) + if axis == "FocusTrack_Z" or scale_mean == 0: + # FocusTrack_Z is not uniform and not necessarily ordered + # Fix me when hyperspy supports non-ordered non-uniform axis + # For now, remove units and fall back on default axis + # nav_dict[axis]["axis"] = data + if scale_mean == 0: + # case "scale_mean == 0" is for series where the axis is invariant. + # In principle, this should happen but the WiRE software allows it + reason = f"Axis {axis} is invariant" + else: + reason = "Non-ordered axis is not supported" + _logger.warning( + f"{reason}, a default axis with scale 1 " + "and offset 0 will be used." + ) + del nav_dict[axis]["units"] + else: + # time axis in test data is not perfectly uniform, but X,Y,Z are + nav_dict[axis]["offset"] = data[0] + nav_dict[axis]["scale"] = scale_mean + return nav_dict def _compare_measurement_type_to_ORGN_WMAP(self, orgn_data, wmap_data): @@ -1144,7 +1149,7 @@ def _reshape_data(self): def _map_general_md(self): general = {} general["title"] = self.original_metadata.get("WDF1_1", {}).get("title") - general["original_filename"] = self._filename + general["original_filename"] = os.path.split(self._filename)[1] try: date, time = self.original_metadata["WDF1_1"]["time_start"].split("#") except KeyError: @@ -1158,7 +1163,7 @@ def _map_signal_md(self): signal = {} if importlib.util.find_spec("lumispy") is None: _logger.warning( - "Cannot find package lumispy, using BaseSignal as signal_type." + "Cannot find package lumispy, using generic signal class BaseSignal." ) signal["signal_type"] = "" else: @@ -1225,6 +1230,8 @@ def map_metadata(self): laser = self._map_laser_md() spectrometer = self._map_spectrometer_md() + # TODO: find laser power? + metadata = { "General": general, "Signal": signal, @@ -1245,53 +1252,72 @@ def _parse_TEXT(self): text = self.__read_utf8(block_size - 16) self.original_metadata.update({"TEXT_0": text}) - def _parse_WHTL(self): + def _get_WHTL(self): if not self._check_block_exists("WHTL_0"): - return + return None pos, size = self._block_info["WHTL_0"] jpeg_header = 0x10 self._file_obj.seek(pos) img_bytes = self._file_obj.read(size - jpeg_header) img = BytesIO(img_bytes) - whtl_metadata = {"image": img} - ## extract EXIF tags and store them in metadata + ## extract and parse EXIF tags if PIL_installed: + from rsciio.utils.image import _parse_axes_from_metadata, _parse_exif_tags + pil_img = Image.open(img) - ## missing header keys when Pillow >= 8.2.0 -> does not flatten IFD anymore - ## see https://pillow.readthedocs.io/en/stable/releasenotes/8.2.0.html#image-getexif-exif-and-gps-ifd - ## Use fall-back _getexif method instead - exif_header = dict(pil_img._getexif()) - whtl_metadata["FocalPlaneResolutionUnit"] = str( - UnitType(exif_header.get(ExifTags.FocalPlaneResolutionUnit)) - ) - whtl_metadata["FocalPlaneXResolution"] = exif_header.get( - ExifTags.FocalPlaneXResolution - ) - whtl_metadata["FocalPlaneYResolution"] = exif_header.get( - ExifTags.FocalPlaneYResolution - ) - whtl_metadata["FocalPlaneXYOrigins"] = exif_header.get( - ExifTags.FocalPlaneXYOrigins - ) - whtl_metadata["ImageDescription"] = exif_header.get( - ExifTags.ImageDescription - ) - whtl_metadata["Make"] = exif_header.get(ExifTags.Make) - whtl_metadata["Unknown"] = exif_header.get(ExifTags.Unknown) - whtl_metadata["FieldOfViewXY"] = exif_header.get(ExifTags.FieldOfViewXY) + original_metadata = {} + data = rgb_tools.regular_array2rgbx(np.array(pil_img)) + original_metadata["exif_tags"] = _parse_exif_tags(pil_img) + axes = _parse_axes_from_metadata(original_metadata["exif_tags"], data.shape) + metadata = { + "General": {"original_filename": os.path.split(self._filename)[1]}, + "Signal": {"signal_type": ""}, + } + + map_md = self.original_metadata.get("WMAP_0") + if map_md is not None: + width = map_md["scale_xyz"][0] * map_md["size_xyz"][0] + length = map_md["scale_xyz"][1] * map_md["size_xyz"][1] + offset = ( + np.array(map_md["offset_xyz"][:2]) + np.array([width, length]) / 2 + ) + + marker_dict = { + "class": "Rectangles", + "name": "Map", + "plot_on_signal": True, + "kwargs": { + "offsets": offset, + "widths": width, + "heights": length, + "color": ("red",), + "facecolor": "none", + }, + } + + metadata["Markers"] = {"Map": marker_dict} - self.original_metadata.update({"WHTL_0": whtl_metadata}) + return { + "axes": axes, + "data": data, + "metadata": metadata, + "original_metadata": original_metadata, + } + else: # pragma: no cover + # Explicit return for readibility + return None def file_reader( filename, lazy=False, - use_uniform_signal_axis=True, + use_uniform_signal_axis=False, load_unmatched_metadata=False, ): """ - Read Renishaw's ``.wdf`` file. + Read Renishaw's ``.wdf`` file. In case of mapping data, the image area will + be returned with a marker showing the mapped area. Parameters ---------- @@ -1332,9 +1358,13 @@ def file_reader( dictionary["metadata"] = deepcopy(wdf.metadata) dictionary["original_metadata"] = deepcopy(wdf.original_metadata) - return [ - dictionary, - ] + image_dict = wdf._get_WHTL() + + dict_list = [dictionary] + if image_dict is not None: + dict_list.append(image_dict) + + return dict_list file_reader.__doc__ %= (FILENAME_DOC, LAZY_DOC, RETURNS_DOC) diff --git a/rsciio/tests/data/image/renishaw_wire.jpg b/rsciio/tests/data/image/renishaw_wire.jpg new file mode 100644 index 000000000..27a24af9b Binary files /dev/null and b/rsciio/tests/data/image/renishaw_wire.jpg differ diff --git a/rsciio/tests/data/renishaw/renishaw_test_focustrack_invariant.wdf b/rsciio/tests/data/renishaw/renishaw_test_focustrack_invariant.wdf new file mode 100644 index 000000000..67b06e007 Binary files /dev/null and b/rsciio/tests/data/renishaw/renishaw_test_focustrack_invariant.wdf differ diff --git a/rsciio/tests/registry.txt b/rsciio/tests/registry.txt index d3573b9a2..c7134027f 100644 --- a/rsciio/tests/registry.txt +++ b/rsciio/tests/registry.txt @@ -175,6 +175,7 @@ 'hspy/test_marker_point_y2_data_deleted.hdf5' 11f24a1d91b3157c12e01929d8bfee9757a5cc29281a6220c13f1638cc3ca49c 'hspy/test_rgba16.hdf5' 5d76658ae9a9416cbdcb239059ee20d640deb634120e1fa051e3199534c47270 'hspy/with_lists_etc.hdf5' 16ed9d4bcb44ba3510963c102eab888b89516921cd4acc4fdb85271407dae562 +'image/renishaw_wire.jpg' 21d34f130568e161a3b2c8a213aa28991880ca0265aec8bfa3c6ca4d9897540c 'impulse/NoMetadata_Synchronized data.csv' 3031a84b6df77f3cfe3808fcf993f3cf95b6a9f67179524200b3129a5de47ef5 'impulse/StubExperiment_Heat raw.csv' 114ebae61321ceed4c071d35e1240a51c2a3bfe37ff9d507cacb7a7dd3977703 'impulse/StubExperiment_Metadata.log' 4b034d75685d61810025586231fb0adfecbbacd171d89230dbf82d54dff7a93c @@ -266,6 +267,7 @@ 'renishaw/renishaw_test_exptime1_acc1.wdf' bc23e1f2644d37dd5b572e587bbcf6db08f33dc7e1480c232b04ef17efa63ba6 'renishaw/renishaw_test_exptime1_acc2.wdf' 7fb5fb09a079d1af672d3d37c5cbf3d950a6d0783791505c6f42d7d104790711 'renishaw/renishaw_test_focustrack.wdf' 73fce4347ece1582afb92cb8cd965e021c825815746037eb7cca7af9133e2350 +'renishaw/renishaw_test_focustrack_invariant.wdf' e2a6d79ab342e7217ed8025c3edd266675112359540bb36a026726bc2513a61a 'renishaw/renishaw_test_linescan.wdf' 631ac664443822e1393b9feef384b5cf80ad53d07c1ce30b9f1136efa8d6d685 'renishaw/renishaw_test_map.wdf' 92f9051e9330c9bb61c5eca1b230c1d05137d8596da490b72a3684dc3665b9fe 'renishaw/renishaw_test_map2.wdf' 72484b2337b9e95676d01b1a6a744a7a82db72af1c58c72ce5b55f07546e49c6 diff --git a/rsciio/tests/test_image.py b/rsciio/tests/test_image.py index 48b4cf09e..4865d271d 100644 --- a/rsciio/tests/test_image.py +++ b/rsciio/tests/test_image.py @@ -17,12 +17,15 @@ # along with RosettaSciIO. If not, see . from packaging.version import Version +from pathlib import Path import numpy as np import pytest imageio = pytest.importorskip("imageio") +testfile_dir = (Path(__file__).parent / "data" / "image").resolve() + from rsciio.image import file_writer @@ -264,3 +267,19 @@ def test_error_library_no_installed(tmp_path): file_writer( tmp_path / "test_image_error.jpg", signal_dict, imshow_kwds={"a": "b"} ) + + +def test_renishaw_wire(): + hs = pytest.importorskip("hyperspy.api", reason="hyperspy not installed") + s = hs.load(testfile_dir / "renishaw_wire.jpg") + assert s.data.shape == (480, 752) + for axis, scale, offset, name in zip( + s.axes_manager.signal_axes, + [2.42207446, 2.503827], + [19105.5, -6814.538], + ["y", "x"], + ): + np.testing.assert_allclose(axis.scale, scale) + np.testing.assert_allclose(axis.offset, offset) + axis.name == name + axis.units == "µm" diff --git a/rsciio/tests/test_renishaw.py b/rsciio/tests/test_renishaw.py index e506fc9c4..745f2f790 100644 --- a/rsciio/tests/test_renishaw.py +++ b/rsciio/tests/test_renishaw.py @@ -39,6 +39,9 @@ testfile_map_block = (testfile_dir / "renishaw_test_map2.wdf").resolve() testfile_timeseries = (testfile_dir / "renishaw_test_timeseries.wdf").resolve() testfile_focustrack = (testfile_dir / "renishaw_test_focustrack.wdf").resolve() +testfile_focustrack_invariant = ( + testfile_dir / "renishaw_test_focustrack_invariant.wdf" +).resolve() testfile_acc1_exptime1 = (testfile_dir / "renishaw_test_exptime1_acc1.wdf").resolve() testfile_acc1_exptime10 = (testfile_dir / "renishaw_test_exptime10_acc1.wdf").resolve() testfile_acc2_exptime1 = (testfile_dir / "renishaw_test_exptime1_acc2.wdf").resolve() @@ -891,7 +894,7 @@ def setup_class(cls): testfile_linescan, reader="Renishaw", use_uniform_signal_axis=True, - ) + )[0] @classmethod def teardown_class(cls): @@ -971,7 +974,7 @@ def setup_class(cls): testfile_map, reader="Renishaw", use_uniform_signal_axis=True, - ) + )[0] @classmethod def teardown_class(cls): @@ -1180,7 +1183,7 @@ def setup_class(cls): testfile_streamline, reader="Renishaw", use_uniform_signal_axis=True, - ) + )[0] @classmethod def teardown_class(cls): @@ -1196,22 +1199,43 @@ def test_data(self): self.s.inav[44, 48].isig[-3:].data, [587.48083, 570.73505, 583.5814] ) - def test_original_metadata_WHTL(self): + def test_WHTL(self): + s = hs.load( + testfile_streamline, + reader="Renishaw", + )[1] expected_WTHL = { - "FocalPlaneResolutionUnit": "µm", + "FocalPlaneResolutionUnit": 5, "FocalPlaneXResolution": 445.75, "FocalPlaneYResolution": 270.85, "FocalPlaneXYOrigins": (-8325.176, -1334.639), + "ExifOffset": 114, "ImageDescription": "white-light image", "Make": "Renishaw", "Unknown": 20.0, "FieldOfViewXY": (8915.0, 5417.0), } - metadata_WHTL = deepcopy(self.s.original_metadata.WHTL_0.as_dictionary()) - metadata_WHTL.pop("image", None) + for i, (axis, scale) in enumerate( + zip(s.axes_manager._axes, (22.570833, 23.710106)) + ): + assert axis.units == "µm" + np.testing.assert_allclose(axis.scale, scale) + np.testing.assert_allclose( + axis.offset, expected_WTHL["FocalPlaneXYOrigins"][::-1][i] + ) + + metadata_WHTL = s.original_metadata.as_dictionary()["exif_tags"] assert metadata_WHTL == expected_WTHL + md = s.metadata.Markers.as_dictionary() + np.testing.assert_allclose( + md["Map"]["kwargs"]["offsets"], + [-8041.7998, -1137.6001], + ) + np.testing.assert_allclose(md["Map"]["kwargs"]["widths"], 116.99999) + np.testing.assert_allclose(md["Map"]["kwargs"]["heights"], 127.39999) + def test_original_metadata_WMAP(self): expected_WMAP = { "linefocus_size": 0, @@ -1263,7 +1287,7 @@ def setup_class(cls): testfile_map_block, reader="Renishaw", use_uniform_signal_axis=True, - ) + )[0] @classmethod def teardown_class(cls): @@ -1347,8 +1371,9 @@ def test_axes(self): assert len(axes_manager) == 2 z_axis = axes_manager.pop("axis-0") - np.testing.assert_allclose(z_axis["scale"], 2.9, atol=0.1) - np.testing.assert_allclose(z_axis["offset"], 26, atol=0.5) + # As hyperspy doesn't support non-ordered axis, default axis are used + np.testing.assert_allclose(z_axis["scale"], 1, atol=0.1) + np.testing.assert_allclose(z_axis["offset"], 0, atol=0.5) def test_data(self): np.testing.assert_allclose( @@ -1360,6 +1385,15 @@ def test_data(self): ) +def test_focus_track_invariant(): + s = hs.load(testfile_focustrack_invariant) + assert s.data.shape == (10, 1010) + z_axis = s.axes_manager[0] + assert z_axis.scale == 1 + assert z_axis.offset == 0 + assert str(z_axis.units) == "" + + class TestPSETMetadata: data_directory = ( Path(__file__).parent / "data" / "renishaw" / "generated_files" diff --git a/rsciio/utils/image.py b/rsciio/utils/image.py new file mode 100644 index 000000000..0d58b1316 --- /dev/null +++ b/rsciio/utils/image.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# Copyright 2007-2024 The HyperSpy developers +# +# This file is part of RosettaSciIO. +# +# RosettaSciIO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# RosettaSciIO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with RosettaSciIO. If not, see . + +from PIL.ExifTags import TAGS + + +CustomTAGS = { + **TAGS, + # Customized EXIF TAGS from Renishaw + 0xFEA0: "FocalPlaneXYOrigins", # 65184 + 0xFEA1: "FieldOfViewXY", # 65185 + 0xFEA2: "Unknown", # 65186 +} + + +# from https://exiftool.org/TagNames/EXIF.html +# For tag 0x9210 (37392) +FocalPlaneResolutionUnit_mapping = { + "": "", + 1: "", + 2: "inches", + 3: "cm", + 4: "mm", + 5: "µm", +} + + +def _parse_axes_from_metadata(exif_tags, sizes): + if exif_tags is None: + return [] + offsets = exif_tags.get("FocalPlaneXYOrigins", [0, 0]) + # jpg files made with Renishaw have this tag + scales = exif_tags.get("FieldOfViewXY", [1, 1]) + + unit = FocalPlaneResolutionUnit_mapping[ + exif_tags.get("FocalPlaneResolutionUnit", "") + ] + + axes = [ + { + "name": name, + "units": unit, + "size": size, + "scale": scales[i] / size, + "offset": offsets[i], + "index_in_array": i, + } + for i, name, size in zip([1, 0], ["y", "x"], sizes) + ] + + return axes + + +def _parse_exif_tags(im): + """ + Parse exif tags from a pillow image + + Parameters + ---------- + im : :class:`PIL.Image` + The pillow image from which the exif tags will be parsed. + + Returns + ------- + exif_dict : None or dict + The dictionary of exif tags. + + """ + exif_dict = None + try: + # missing header keys when Pillow >= 8.2.0 -> does not flatten IFD anymore + # see https://pillow.readthedocs.io/en/stable/releasenotes/8.2.0.html#image-getexif-exif-and-gps-ifd + # Use fall-back _getexif method instead + # Not all format plugin have the private method + # prefer to use that method as it returns more items + exif_dict = im._getexif() + except AttributeError: + exif_dict = im.getexif() + if exif_dict is not None: + exif_dict = {CustomTAGS.get(k, "unknown"): v for k, v in exif_dict.items()} + + return exif_dict diff --git a/upcoming_changes/227.enhancements.rst b/upcoming_changes/227.enhancements.rst new file mode 100644 index 000000000..56e577a57 --- /dev/null +++ b/upcoming_changes/227.enhancements.rst @@ -0,0 +1,5 @@ +:ref:`Renishaw wdf `: + +- return survey image instead of saving it to the metadata and add marker of the mapping area on the survey image. +- Add support for reading data with invariant axis, for example when the values of the Z axis doesn't change. +- Parse calibration of ``jpg`` images saved with Renishaw Wire software. \ No newline at end of file