Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Compress tiff files #2926

Merged
merged 5 commits into from Apr 15, 2022
Merged

Compress tiff files #2926

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b68c8a5
Compress TVIPS file
ericpre Apr 14, 2022
21e7bf0
Compress tif Zeiss SEM file
ericpre Apr 14, 2022
65b9b19
Compress FEI Helios file
ericpre Apr 14, 2022
52c97fa
Compress test_rgba16 files
ericpre Apr 14, 2022
97d0b5b
Add changelog entry
ericpre Apr 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Binary file removed hyperspy/tests/io/npy_files/test_rgba16.npy
View file
Open in desktop
Binary file not shown.
Binary file added hyperspy/tests/io/npz_files/test_rgba16.npz
View file
Open in desktop
Binary file not shown.
2 changes: 1 addition & 1 deletion hyperspy/tests/io/test_hdf5.py
View file
Open in desktop
Expand Up @@ -403,7 +403,7 @@ def test_rgba16():
with pytest.warns(VisibleDeprecationWarning):
print(my_path)
s = load(my_path / "hdf5_files" / "test_rgba16.hdf5")
data = np.load(my_path / "npy_files" / "test_rgba16.npy")
data = np.load(my_path / "npz_files" / "test_rgba16.npz")['a']
assert (s.data == data).all()


Expand Down
267 changes: 152 additions & 115 deletions hyperspy/tests/io/test_tiff.py
View file
Open in desktop
@@ -1,24 +1,37 @@
import os
from packaging.version import Version
from pathlib import Path
import tempfile
import zipfile


import numpy as np
import pytest
import traits.api as t
from packaging.version import Version
import tifffile


import hyperspy.api as hs
from hyperspy.misc.test_utils import assert_deep_almost_equal
from hyperspy import __version__ as hs_version


MY_PATH = os.path.dirname(__file__)
MY_PATH2 = os.path.join(MY_PATH, "tiff_files")
TMP_DIR = tempfile.TemporaryDirectory()


def teardown_module():
TMP_DIR.cleanup()


def test_rgba16():
s = hs.load(os.path.join(MY_PATH2, "test_rgba16.tif"))
data = np.load(os.path.join(MY_PATH, "npy_files", "test_rgba16.npy"))
path = Path(TMP_DIR.name)
zipf = os.path.join(MY_PATH2, "test_rgba16.zip")
with zipfile.ZipFile(zipf, 'r') as zipped:
zipped.extractall(path)

s = hs.load(path / 'test_rgba16.tif')
data = np.load(Path(MY_PATH) / "npz_files" / "test_rgba16.npz")['a']
assert (s.data == data).all()
assert s.axes_manager[0].units == t.Undefined
assert s.axes_manager[1].units == t.Undefined
Expand Down Expand Up @@ -402,116 +415,135 @@ def test_saving_loading_stack_no_scale():
'unfolded': False}}}


def test_read_FEI_SEM_scale_metadata_8bits():
fname = os.path.join(MY_PATH2, 'FEI-Helios-Ebeam-8bits.tif')
s = hs.load(fname, convert_units=True)
assert s.axes_manager[0].units == 'µm'
assert s.axes_manager[1].units == 'µm'
np.testing.assert_allclose(s.axes_manager[0].scale, 3.3724, rtol=1E-5)
np.testing.assert_allclose(s.axes_manager[1].scale, 3.3724, rtol=1E-5)
assert s.data.dtype == 'uint8'
# delete timestamp from metadata since it's runtime dependent
del s.metadata.General.FileIO.Number_0.timestamp
FEI_Helios_metadata['General'][
'original_filename'] = 'FEI-Helios-Ebeam-8bits.tif'
assert_deep_almost_equal(s.metadata.as_dictionary(), FEI_Helios_metadata)


def test_read_FEI_SEM_scale_metadata_16bits():
fname = os.path.join(MY_PATH2, 'FEI-Helios-Ebeam-16bits.tif')
s = hs.load(fname, convert_units=True)
assert s.axes_manager[0].units == 'µm'
assert s.axes_manager[1].units == 'µm'
np.testing.assert_allclose(s.axes_manager[0].scale, 3.3724, rtol=1E-5)
np.testing.assert_allclose(s.axes_manager[1].scale, 3.3724, rtol=1E-5)
assert s.data.dtype == 'uint16'
# delete timestamp from metadata since it's runtime dependent
del s.metadata.General.FileIO.Number_0.timestamp
FEI_Helios_metadata['General'][
'original_filename'] = 'FEI-Helios-Ebeam-16bits.tif'
assert_deep_almost_equal(s.metadata.as_dictionary(), FEI_Helios_metadata)


def test_read_Zeiss_SEM_scale_metadata_1k_image():
md = {'Acquisition_instrument': {'SEM': {'Stage': {'rotation': 10.2,
'tilt': -0.0,
'x': 75.6442,
'y': 60.4901,
'z': 25.193},
'Detector':{'detector_type':
'HE-SE2'},
'beam_current': 1.0,
'beam_energy': 25.0,
'dwell_time': 5e-08,
'magnification': 105.0,
'microscope': 'Merlin-61-08',
'working_distance': 14.808}},
'General': {'authors': 'LIM',
'date': '2015-12-23',
'original_filename': 'test_tiff_Zeiss_SEM_1k.tif',
'time': '09:40:32',
'title': '',
'FileIO': {'0': {
'operation': 'load',
'hyperspy_version': hs_version,
'io_plugin': 'hyperspy.io_plugins.tiff'
}}
},
'Signal': {'signal_type': ''},
'_HyperSpy': {'Folding': {'original_axes_manager': None,
'original_shape': None,
'signal_unfolded': False,
'unfolded': False}}}
class TestReadFEIHelios:

path = Path(TMP_DIR.name)

@classmethod
def setup_class(cls):
zipf = os.path.join(MY_PATH2, "tiff_FEI_Helios.zip")
with zipfile.ZipFile(zipf, 'r') as zipped:
zipped.extractall(cls.path)

fname = os.path.join(MY_PATH2, 'test_tiff_Zeiss_SEM_1k.tif')
s = hs.load(fname, convert_units=True)
assert s.axes_manager[0].units == 'µm'
assert s.axes_manager[1].units == 'µm'
np.testing.assert_allclose(s.axes_manager[0].scale, 2.614514, rtol=1E-6)
np.testing.assert_allclose(s.axes_manager[1].scale, 2.614514, rtol=1E-6)
assert s.data.dtype == 'uint8'
# delete timestamp from metadata since it's runtime dependent
del s.metadata.General.FileIO.Number_0.timestamp
assert_deep_almost_equal(s.metadata.as_dictionary(), md)


def test_read_Zeiss_SEM_scale_metadata_512_image():
md = {'Acquisition_instrument': {'SEM': {'Stage': {'rotation': 245.8,
'tilt': 0.0,
'x': 62.9961,
'y': 65.3168,
'z': 44.678},
'beam_energy': 5.0,
'magnification': '50.00 K X',
'microscope': 'ULTRA 55-36-06',
'working_distance': 3.9}},
'General': {'authors': 'LIBERATO',
'date': '2018-09-25',
'original_filename': 'test_tiff_Zeiss_SEM_512pix.tif',
'time': '08:20:42',
'title': '',
'FileIO': {'0': {
'operation': 'load',
'hyperspy_version': hs_version,
'io_plugin': 'hyperspy.io_plugins.tiff'
}}
},
'Signal': {'signal_type': ''},
'_HyperSpy': {'Folding': {'original_axes_manager': None,
'original_shape': None,
'signal_unfolded': False,
'unfolded': False}}}

fname = os.path.join(MY_PATH2, 'test_tiff_Zeiss_SEM_512pix.tif')
s = hs.load(fname, convert_units=True)
assert s.axes_manager[0].units == 'µm'
assert s.axes_manager[1].units == 'µm'
np.testing.assert_allclose(s.axes_manager[0].scale, 0.011649976, rtol=1E-6)
np.testing.assert_allclose(s.axes_manager[1].scale, 0.011649976, rtol=1E-6)
assert s.data.dtype == 'uint8'
# delete timestamp from metadata since it's runtime dependent
del s.metadata.General.FileIO.Number_0.timestamp
assert_deep_almost_equal(s.metadata.as_dictionary(), md)
def test_read_FEI_SEM_scale_metadata_8bits(self):
fname = self.path / 'FEI-Helios-Ebeam-8bits.tif'
s = hs.load(fname, convert_units=True)
assert s.axes_manager[0].units == 'µm'
assert s.axes_manager[1].units == 'µm'
np.testing.assert_allclose(s.axes_manager[0].scale, 3.3724, rtol=1E-5)
np.testing.assert_allclose(s.axes_manager[1].scale, 3.3724, rtol=1E-5)
assert s.data.dtype == 'uint8'
# delete timestamp from metadata since it's runtime dependent
del s.metadata.General.FileIO.Number_0.timestamp
FEI_Helios_metadata['General'][
'original_filename'] = 'FEI-Helios-Ebeam-8bits.tif'
assert_deep_almost_equal(s.metadata.as_dictionary(), FEI_Helios_metadata)

def test_read_FEI_SEM_scale_metadata_16bits(self):
fname = self.path / 'FEI-Helios-Ebeam-16bits.tif'
s = hs.load(fname, convert_units=True)
assert s.axes_manager[0].units == 'µm'
assert s.axes_manager[1].units == 'µm'
np.testing.assert_allclose(s.axes_manager[0].scale, 3.3724, rtol=1E-5)
np.testing.assert_allclose(s.axes_manager[1].scale, 3.3724, rtol=1E-5)
assert s.data.dtype == 'uint16'
# delete timestamp from metadata since it's runtime dependent
del s.metadata.General.FileIO.Number_0.timestamp
FEI_Helios_metadata['General'][
'original_filename'] = 'FEI-Helios-Ebeam-16bits.tif'
assert_deep_almost_equal(s.metadata.as_dictionary(), FEI_Helios_metadata)


class TestReadZeissSEM:

path = Path(TMP_DIR.name)

@classmethod
def setup_class(cls):
zipf = os.path.join(MY_PATH2, "tiff_Zeiss_SEM.zip")
with zipfile.ZipFile(zipf, 'r') as zipped:
zipped.extractall(cls.path)

def test_read_Zeiss_SEM_scale_metadata_1k_image(self):
md = {'Acquisition_instrument': {'SEM': {'Stage': {'rotation': 10.2,
'tilt': -0.0,
'x': 75.6442,
'y': 60.4901,
'z': 25.193},
'Detector':{'detector_type':
'HE-SE2'},
'beam_current': 1.0,
'beam_energy': 25.0,
'dwell_time': 5e-08,
'magnification': 105.0,
'microscope': 'Merlin-61-08',
'working_distance': 14.808}},
'General': {'authors': 'LIM',
'date': '2015-12-23',
'original_filename': 'test_tiff_Zeiss_SEM_1k.tif',
'time': '09:40:32',
'title': '',
'FileIO': {'0': {
'operation': 'load',
'hyperspy_version': hs_version,
'io_plugin': 'hyperspy.io_plugins.tiff'
}}
},
'Signal': {'signal_type': ''},
'_HyperSpy': {'Folding': {'original_axes_manager': None,
'original_shape': None,
'signal_unfolded': False,
'unfolded': False}}}

fname = self.path / 'test_tiff_Zeiss_SEM_1k.tif'
s = hs.load(fname, convert_units=True)

assert s.axes_manager[0].units == 'µm'
assert s.axes_manager[1].units == 'µm'
np.testing.assert_allclose(s.axes_manager[0].scale, 2.614514, rtol=1E-6)
np.testing.assert_allclose(s.axes_manager[1].scale, 2.614514, rtol=1E-6)
assert s.data.dtype == 'uint8'
# delete timestamp from metadata since it's runtime dependent
del s.metadata.General.FileIO.Number_0.timestamp
assert_deep_almost_equal(s.metadata.as_dictionary(), md)

def test_read_Zeiss_SEM_scale_metadata_512_image(self):
md = {'Acquisition_instrument': {'SEM': {'Stage': {'rotation': 245.8,
'tilt': 0.0,
'x': 62.9961,
'y': 65.3168,
'z': 44.678},
'beam_energy': 5.0,
'magnification': '50.00 K X',
'microscope': 'ULTRA 55-36-06',
'working_distance': 3.9}},
'General': {'authors': 'LIBERATO',
'date': '2018-09-25',
'original_filename': 'test_tiff_Zeiss_SEM_512pix.tif',
'time': '08:20:42',
'title': '',
'FileIO': {'0': {
'operation': 'load',
'hyperspy_version': hs_version,
'io_plugin': 'hyperspy.io_plugins.tiff'
}}
},
'Signal': {'signal_type': ''},
'_HyperSpy': {'Folding': {'original_axes_manager': None,
'original_shape': None,
'signal_unfolded': False,
'unfolded': False}}}

fname = self.path / 'test_tiff_Zeiss_SEM_512pix.tif'
s = hs.load(fname, convert_units=True)
assert s.axes_manager[0].units == 'µm'
assert s.axes_manager[1].units == 'µm'
np.testing.assert_allclose(s.axes_manager[0].scale, 0.011649976, rtol=1E-6)
np.testing.assert_allclose(s.axes_manager[1].scale, 0.011649976, rtol=1E-6)
assert s.data.dtype == 'uint8'
# delete timestamp from metadata since it's runtime dependent
del s.metadata.General.FileIO.Number_0.timestamp
assert_deep_almost_equal(s.metadata.as_dictionary(), md)


def test_read_RGB_Zeiss_optical_scale_metadata():
Expand Down Expand Up @@ -604,8 +636,13 @@ def test_read_TVIPS_metadata():
'original_shape': None,
'signal_unfolded': False,
'unfolded': False}}}
fname = os.path.join(MY_PATH2, 'TVIPS_bin4.tif')
s = hs.load(fname, convert_units=True)

zipf = os.path.join(MY_PATH2, "TVIPS_bin4.zip")
path = Path(TMP_DIR.name)
with zipfile.ZipFile(zipf, 'r') as zipped:
zipped.extractall(path)
s = hs.load(path / 'TVIPS_bin4.tif', convert_units=True)

assert s.data.dtype == np.uint8
assert s.data.shape == (1024, 1024)
assert s.axes_manager[0].units == 'nm'
Expand Down
Binary file removed hyperspy/tests/io/tiff_files/FEI-Helios-Ebeam-16bits.tif
View file
Open in desktop
Binary file not shown.
Binary file removed hyperspy/tests/io/tiff_files/FEI-Helios-Ebeam-8bits.tif
View file
Open in desktop
Binary file not shown.
Binary file removed hyperspy/tests/io/tiff_files/TVIPS_bin4.tif
View file
Open in desktop
Binary file not shown.
Binary file added hyperspy/tests/io/tiff_files/TVIPS_bin4.zip
View file
Open in desktop
Binary file not shown.
Binary file removed hyperspy/tests/io/tiff_files/test_rgba16.tif
View file
Open in desktop
Binary file not shown.
Binary file added hyperspy/tests/io/tiff_files/test_rgba16.zip
View file
Open in desktop
Binary file not shown.
Binary file removed hyperspy/tests/io/tiff_files/test_tiff_Zeiss_SEM_1k.tif
View file
Open in desktop
Binary file not shown.
Binary file removed hyperspy/tests/io/tiff_files/test_tiff_Zeiss_SEM_512pix.tif
View file
Open in desktop
Binary file not shown.
Binary file added hyperspy/tests/io/tiff_files/tiff_FEI_Helios.zip
View file
Open in desktop
Binary file not shown.
Binary file added hyperspy/tests/io/tiff_files/tiff_Zeiss_SEM.zip
View file
Open in desktop
Binary file not shown.
3 changes: 2 additions & 1 deletion setup.py
View file
Open in desktop
Expand Up @@ -362,10 +362,11 @@ def __exit__(self, type, value, traceback):
'tests/io/JEOL_files/Sample/00_View000/*',
'tests/io/JEOL_files/InvalidFrame/*',
'tests/io/JEOL_files/InvalidFrame/Sample/00_Dummy-Data/*',
'tests/io/tiff_files/*.zip',
'tests/io/tiff_files/*.tif',
'tests/io/tiff_files/*.tif.gz',
'tests/io/tiff_files/*.dm3',
'tests/io/npy_files/*.npy',
'tests/io/npz_files/*.npz',
'tests/io/unf_files/*.unf',
'tests/io/bruker_data/*.bcf',
'tests/io/bruker_data/*.json',
Expand Down
1 change: 1 addition & 0 deletions upcoming_changes/2926.maintenance.rst
View file
Open in desktop
@@ -0,0 +1 @@
Compress some tiff tests files to reduce package size