From ecd36fb7c63c5d4386637a67c49f450e41f35dbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Wiik=20=C3=85nes?= Date: Fri, 20 Sep 2019 17:04:04 +0200 Subject: [PATCH 1/5] Fix NORDIF read. sig/nav size params overwrite + start io-tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Håkon Wiik Ånes --- kikuchipy/io_plugins/nordif.py | 14 +-- kikuchipy/io_plugins/test/__init__.py | 17 ++++ kikuchipy/io_plugins/test/test_nordif.py | 119 +++++++++++++++++++++++ 3 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 kikuchipy/io_plugins/test/__init__.py create mode 100644 kikuchipy/io_plugins/test/test_nordif.py diff --git a/kikuchipy/io_plugins/nordif.py b/kikuchipy/io_plugins/nordif.py index d83204bc..c1f76ac1 100644 --- a/kikuchipy/io_plugins/nordif.py +++ b/kikuchipy/io_plugins/nordif.py @@ -87,8 +87,10 @@ def file_reader(filename, mmap_mode=None, scan_size=None, setting_file_exists = os.path.isfile(setting_file) if setting_file_exists: md, omd, scan_size_file = get_settings_from_file(setting_file) - scan_size = (scan_size_file.nx, scan_size_file.ny) - pattern_size = (scan_size_file.sx, scan_size_file.sy) + if not scan_size: + scan_size = (scan_size_file.nx, scan_size_file.ny) + if not pattern_size: + pattern_size = (scan_size_file.sx, scan_size_file.sy) else: warnings.warn("No setting file found, will attempt to use values for " "scan_size and pattern_size from input arguments.") @@ -145,7 +147,7 @@ def file_reader(filename, mmap_mode=None, scan_size=None, # Calibrate scan dimension try: scales[:2] = scales[:2] * scan_size_file.step_x - except TypeError: + except (TypeError, UnboundLocalError): warnings.warn("Could not calibrate scan dimensions, this can be done " "using set_scan_calibration()") @@ -283,9 +285,9 @@ def get_string(content, expression, line_no, file): match = re.search(expression, content[line_no]) if match is None: - warnings.warn("Failed to read line {} in settings file '{}' using " - "regular expression '{}'".format(line_no - 1, - file.name, expression)) + warnings.warn( + "Failed to read line {} in settings file '{}' using regular " + "expression '{}'".format(line_no - 1, file.name, expression)) return 0 else: return match.group(1) diff --git a/kikuchipy/io_plugins/test/__init__.py b/kikuchipy/io_plugins/test/__init__.py new file mode 100644 index 00000000..699d1be6 --- /dev/null +++ b/kikuchipy/io_plugins/test/__init__.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 The KikuchiPy developers +# +# This file is part of KikuchiPy. +# +# KikuchiPy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# KikuchiPy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with KikuchiPy. If not, see . diff --git a/kikuchipy/io_plugins/test/test_nordif.py b/kikuchipy/io_plugins/test/test_nordif.py new file mode 100644 index 00000000..7a5560fe --- /dev/null +++ b/kikuchipy/io_plugins/test/test_nordif.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 The KikuchiPy developers +# +# This file is part of KikuchiPy. +# +# KikuchiPy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# KikuchiPy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with KikuchiPy. If not, see . + +import os +import pytest +import tempfile +import gc +import numpy as np +from kikuchipy.io_plugins.nordif import ( + file_reader, get_settings_from_file, get_string, file_writer) + + +DIR_PATH = os.path.dirname(__file__) +NORDIF_PATH = os.path.join(DIR_PATH, '../../data/nordif') + +# Settings content +METADATA = { + 'Acquisition_instrument': { + 'SEM': { + 'microscope': 'Hitachi SU-6600', 'magnification': 200, + 'beam_energy': 20.0, 'working_distance': 24.7, 'Detector': { + 'EBSD': { + 'azimuth_angle': 0.0, 'binning': 1, + 'detector': 'NORDIF UF1100', 'elevation_angle': 0.0, + 'exposure_time': 0.0035, 'frame_number': 1, + 'frame_rate': 202, 'gain': 0.0, 'grid_type': 'square', + 'sample_tilt': 70.0, 'scan_time': 148, + 'static_background': 1, 'xpc': 1.0, 'ypc': 1.0, 'zpc': 1.0, + 'version': '3.1.2', 'manufacturer': 'NORDIF'}}}}, + 'Sample': { + 'Phases': { + '1': { + 'atom_coordinates': { + '1': { + 'atom': '', 'coordinates': np.array([0., 0., 0.]), + 'site_occupation': 0.0, + 'debye_waller_factor': 0.0}}, + 'formula': '', 'info': '', + 'lattice_constants': np.array([0., 0., 0., 0., 0., 0.]), + 'laue_group': '', 'material_name': 'Ni', 'point_group': '', + 'setting': 0, 'space_group': 0, 'symmetry': 0}}}} +ORIGINAL_METADATA = { + 'nordif_header': [ + '[NORDIF]\t\t', 'Software version\t3.1.2\t', '\t\t', + '[Microscope]\t\t', 'Manufacturer\tHitachi\t', 'Model\tSU-6600\t', + 'Magnification\t200\t#', 'Scan direction\tDirect\t', + 'Accelerating voltage\t20\tkV', 'Working distance\t24.7\tmm', + 'Tilt angle\t70\t°', '\t\t', '[Signal voltages]\t\t', + 'Minimum\t0.0\tV', 'Maximum\t1.0\tV', '\t\t', + '[Deflection voltages]\t\t', 'Minimum\t-5.5\tV', 'Maximum\t5.5\tV', + '\t\t', '[Electron image]\t\t', 'Frame rate\t0.25\tfps', + 'Resolution\t1000x1000\tpx', 'Rotation\t0\t°', 'Flip x-axis\tFalse\t', + 'Flip y-axis\tFalse\t', 'Calibration factor\t7273\tµm/V', + 'Tilt axis\tx-axis\t', '\t\t', '[Aspect ratio]\t\t', + 'X-axis\t1.000\t', 'Y-axis\t1.000\t', '\t\t', '[EBSD detector]\t\t', + 'Model\tUF1100\t', 'Port position\t90\t', 'Jumbo frames\tFalse\t', + '\t\t', '[Detector angles]\t\t', 'Euler 1\t0\t°', 'Euler 2\t0\t°', + 'Euler 3\t0\t°', 'Azimuthal\t0\t°', 'Elevation\t0\t°', '\t\t', + '[Acquisition settings]\t\t', 'Frame rate\t202\tfps', + 'Resolution\t60x60\tpx', 'Exposure time\t3500\tµs', 'Gain\t0\t', '\t\t', + '[Calibration settings]\t\t', 'Frame rate\t10\tfps', + 'Resolution\t480x480\tpx', 'Exposure time\t99950\tµs', 'Gain\t8\t', + '\t\t', '[Specimen]\t\t', 'Name\tNi\t', + 'Mounting\t1. ND||EB TD||TA\t', '\t\t', '[Phase 1]\t\t', 'Name\t\t', + 'Pearson S.T.\t\t', 'IT\t\t', '\t\t', '[Phase 2]\t\t', 'Name\t\t', + 'Pearson S.T.\t\t', 'IT\t\t', '\t\t', '[Region of interest]\t\t', + '\t\t', '[Area]\t\t', 'Top\t89.200 (223)\tµm (px)', + 'Left\t60.384 (152)\tµm (px)', 'Width\t4.500 (11)\tµm (px)', + 'Height\t4.500 (11)\tµm (px)', 'Step size\t1.500\tµm', + 'Number of samples\t3x3\t#', 'Scan time\t00:02:28\t', '\t\t', + '[Points of interest]\t\t', '\t\t', '[Acquisition patterns]\t\t', + 'Acquisition (507,500)\t507,500\tpx', + 'Acquisition (393,501)\t393,501\tpx', + 'Acquisition (440,448)\t440,448\tpx', '\t\t', + '[Calibration patterns]\t\t', 'Calibration (425,447)\t425,447\tpx', + 'Calibration (294,532)\t294,532\tpx', + 'Calibration (573,543)\t573,543\tpx', + 'Calibration (596,378)\t596,378\tpx', + 'Calibration (308,369)\t308,369\tpx', + 'Calibration (171,632)\t171,632\tpx', + 'Calibration (704,668)\t704,668\tpx', + 'Calibration (696,269)\t696,269\tpx', + 'Calibration (152,247)\t152,247\tpx']} +AXES_MANAGER = { + 'nx': 3, 'ny': 3, 'sx': 60, 'sy': 60, 'step_x': 1.5, 'step_y': 1.5} + + +@pytest.fixture() +def save_path(): + with tempfile.TemporaryDirectory() as tmp: + file_path = os.path.join(tmp, 'nordif', 'save_temp.dat') + yield file_path + gc.collect() + + +class TestNORDIF: + + def test_get_settings_from_file(self): + setting_file = os.path.join(NORDIF_PATH, 'Setting.txt') + settings = get_settings_from_file(setting_file) + answers = [METADATA, ORIGINAL_METADATA, AXES_MANAGER] + assert len(settings) == len(answers) + for setting_read, answer in zip(settings, answers): + np.testing.assert_equal(setting_read.as_dictionary(), answer) From 2e64f06e050fa0da5d9992545e0b9ffef73c4197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Wiik=20=C3=85nes?= Date: Sat, 21 Sep 2019 20:59:26 +0200 Subject: [PATCH 2/5] Add complete tests for NORDIF reader/writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Håkon Wiik Ånes --- .../data/nordif/{patterns.dat => Pattern.dat} | 0 kikuchipy/io_plugins/h5ebsd.py | 2 +- kikuchipy/io_plugins/nordif.py | 73 +++++---- kikuchipy/io_plugins/test/test_nordif.py | 153 +++++++++++++++++- 4 files changed, 189 insertions(+), 39 deletions(-) rename kikuchipy/data/nordif/{patterns.dat => Pattern.dat} (100%) diff --git a/kikuchipy/data/nordif/patterns.dat b/kikuchipy/data/nordif/Pattern.dat similarity index 100% rename from kikuchipy/data/nordif/patterns.dat rename to kikuchipy/data/nordif/Pattern.dat diff --git a/kikuchipy/io_plugins/h5ebsd.py b/kikuchipy/io_plugins/h5ebsd.py index 8e1ac9f1..2f89d36b 100644 --- a/kikuchipy/io_plugins/h5ebsd.py +++ b/kikuchipy/io_plugins/h5ebsd.py @@ -45,7 +45,7 @@ file_extensions = ['h5', 'hdf5', 'h5ebsd'] default_extension = 0 # Writing capabilities -writes = [(2, 2), (2, 1), (2, 0)] +writes = [(2, 2), (2, 1)] def file_reader(filename, scans=None, lazy=False, **kwargs): diff --git a/kikuchipy/io_plugins/nordif.py b/kikuchipy/io_plugins/nordif.py index c1f76ac1..517b9f2c 100644 --- a/kikuchipy/io_plugins/nordif.py +++ b/kikuchipy/io_plugins/nordif.py @@ -40,8 +40,9 @@ writes = [(2, 2), (2, 1), (2, 0)] -def file_reader(filename, mmap_mode=None, scan_size=None, - pattern_size=None, setting_file=None, lazy=False): +def file_reader( + filename, mmap_mode=None, scan_size=None, pattern_size=None, + setting_file=None, lazy=False): """Read electron backscatter patterns from a NORDIF data file. Parameters @@ -49,7 +50,7 @@ def file_reader(filename, mmap_mode=None, scan_size=None, filename : str File path to NORDIF data file. mmap_mode : str, optional - scan_size : None or tuple, optional + scan_size : {None, int, or tuple}, optional Scan size in number of patterns in width and height. pattern_size : None or tuple, optional Pattern size in detector pixels in width and height. @@ -70,8 +71,8 @@ def file_reader(filename, mmap_mode=None, scan_size=None, scan = {'attributes': {}} # Make sure we open in correct mode - if '+' in mmap_mode or ('write' in mmap_mode and - 'copyonwrite' != mmap_mode): + if '+' in mmap_mode or ( + 'write' in mmap_mode and 'copyonwrite' != mmap_mode): if lazy: raise ValueError("Lazy loading does not support in-place writing") f = open(filename, 'r+b') @@ -92,8 +93,11 @@ def file_reader(filename, mmap_mode=None, scan_size=None, if not pattern_size: pattern_size = (scan_size_file.sx, scan_size_file.sy) else: - warnings.warn("No setting file found, will attempt to use values for " - "scan_size and pattern_size from input arguments.") + if scan_size is None and pattern_size is None: + raise ValueError( + "No setting file found and no scan_size or pattern_size " + "detected in input arguments. These must be set if no setting " + "file is provided.") md = kikuchipy_metadata() omd = DictionaryTreeBrowser() @@ -103,21 +107,25 @@ def file_reader(filename, mmap_mode=None, scan_size=None, md.set_item(ebsd_node + '.static_background', plt.imread(static_bg_file)) except FileNotFoundError: - warnings.warn("Could not read static background pattern '{}', however " - "it can be added using set_experimental_parameters()." - "".format(static_bg_file)) + warnings.warn( + "Could not read static background pattern '{}', however it can be " + "added using set_experimental_parameters().".format(static_bg_file)) # Set required and other parameters in metadata md.set_item('General.original_filename', filename) - md.set_item('General.title', - os.path.splitext(os.path.split(filename)[1])[0]) + md.set_item( + 'General.title', os.path.splitext(os.path.split(filename)[1])[0]) md.set_item('Signal.signal_type', 'EBSD') md.set_item('Signal.record_by', 'image') scan['metadata'] = md.as_dictionary() scan['original_metadata'] = omd.as_dictionary() # Set scan size and pattern size - nx, ny = scan_size + if isinstance(scan_size, int): + nx = scan_size + ny = 1 + else: + nx, ny = scan_size sx, sy = pattern_size # Read data from file @@ -131,9 +139,9 @@ def file_reader(filename, mmap_mode=None, scan_size=None, try: data = data.reshape((ny, nx, sy, sx), order='C').squeeze() except ValueError: - warnings.warn("Pattern size and scan size larger than file size! " - "Will attempt to load by zero padding incomplete " - "frames.") + warnings.warn( + "Pattern size and scan size larger than file size! Will attempt to " + "load by zero padding incomplete frames.") # Data is stored pattern by pattern pw = [(0, ny * nx * sy * sx - data.size)] data = np.pad(data, pw, mode='constant') @@ -148,13 +156,15 @@ def file_reader(filename, mmap_mode=None, scan_size=None, try: scales[:2] = scales[:2] * scan_size_file.step_x except (TypeError, UnboundLocalError): - warnings.warn("Could not calibrate scan dimensions, this can be done " - "using set_scan_calibration()") + warnings.warn( + "Could not calibrate scan dimensions, this can be done using " + "set_scan_calibration()") # Create axis objects for each axis - axes = [{'size': data.shape[i], 'index_in_array': i, 'name': names[i], - 'scale': scales[i], 'offset': 0.0, 'units': units[i]} - for i in range(data.ndim)] + axes = [{ + 'size': data.shape[i], 'index_in_array': i, 'name': names[i], + 'scale': scales[i], 'offset': 0.0, 'units': units[i]} + for i in range(data.ndim)] scan['axes'] = axes f.close() @@ -184,9 +194,10 @@ def get_settings_from_file(filename): content = f.read().splitlines() # Get line numbers of setting blocks - blocks = {'[NORDIF]': -1, '[Microscope]': -1, '[EBSD detector]': -1, - '[Detector angles]': -1, '[Acquisition settings]': -1, - '[Area]': -1, '[Specimen]' : -1} + blocks = { + '[NORDIF]': -1, '[Microscope]': -1, '[EBSD detector]': -1, + '[Detector angles]': -1, '[Acquisition settings]': -1, '[Area]': -1, + '[Specimen]' : -1} for i, line in enumerate(content): for block in blocks: if block in line: @@ -208,8 +219,8 @@ def get_settings_from_file(filename): # Get metadata values from settings file using regular expressions azimuth_angle = get_string(content, 'Azimuthal\t(.*)\t', l_ang + 4, f) md.set_item(ebsd_node + '.azimuth_angle', float(azimuth_angle)) - beam_energy = get_string(content, 'Accelerating voltage\t(.*)\tkV', - l_mic + 5, f) + beam_energy = get_string( + content, 'Accelerating voltage\t(.*)\tkV', l_mic + 5, f) md.set_item(sem_node + '.beam_energy', float(beam_energy)) detector = get_string(content, 'Model\t(.*)\t', l_det + 1, f) detector = re.sub('[^a-zA-Z0-9]', repl='', string=detector) @@ -231,14 +242,14 @@ def get_settings_from_file(filename): md.set_item(ebsd_node + '.sample_tilt', float(sample_tilt)) scan_time = get_string(content, 'Scan time\t(.*)\t', l_area + 7, f) scan_time = time.strptime(scan_time, '%H:%M:%S') - scan_time = datetime.timedelta(hours=scan_time.tm_hour, - minutes=scan_time.tm_min, - seconds=scan_time.tm_sec).total_seconds() + scan_time = datetime.timedelta( + hours=scan_time.tm_hour, minutes=scan_time.tm_min, + seconds=scan_time.tm_sec).total_seconds() md.set_item(ebsd_node + '.scan_time', int(scan_time)) version = get_string(content, 'Software version\t(.*)\t', l_nordif + 1, f) md.set_item(ebsd_node + '.version', version) - working_distance = get_string(content, 'Working distance\t(.*)\tmm', - l_mic + 6, f) + working_distance = get_string( + content, 'Working distance\t(.*)\tmm', l_mic + 6, f) md.set_item(sem_node + '.working_distance', float(working_distance)) md.set_item(ebsd_node + '.grid_type', 'square') md.set_item(ebsd_node + '.manufacturer', 'NORDIF') diff --git a/kikuchipy/io_plugins/test/test_nordif.py b/kikuchipy/io_plugins/test/test_nordif.py index 7a5560fe..ebf0378d 100644 --- a/kikuchipy/io_plugins/test/test_nordif.py +++ b/kikuchipy/io_plugins/test/test_nordif.py @@ -16,17 +16,28 @@ # You should have received a copy of the GNU General Public License # along with KikuchiPy. If not, see . +# Many of these tests are inspired by the tests written for the block_file +# reader/writer available in HyperSpy: https://github.com/hyperspy/hyperspy/ +# blob/RELEASE_next_minor/hyperspy/tests/io/test_blockfile.py + import os import pytest import tempfile import gc +import time +import datetime import numpy as np -from kikuchipy.io_plugins.nordif import ( - file_reader, get_settings_from_file, get_string, file_writer) +import dask.array as da +import matplotlib.pyplot as plt +import kikuchipy as kp +from kikuchipy.io_plugins.nordif import get_settings_from_file DIR_PATH = os.path.dirname(__file__) -NORDIF_PATH = os.path.join(DIR_PATH, '../../data/nordif') +PATTERN_FILE = os.path.join(DIR_PATH, '../../data/nordif/Pattern.dat') +SETTING_FILE = os.path.join(DIR_PATH, '../../data/nordif/Setting.txt') +BG_FILE = os.path.join( + DIR_PATH, '../../data/nordif/Background acquisition pattern.bmp') # Settings content METADATA = { @@ -96,9 +107,23 @@ 'Calibration (704,668)\t704,668\tpx', 'Calibration (696,269)\t696,269\tpx', 'Calibration (152,247)\t152,247\tpx']} -AXES_MANAGER = { +SCAN_SIZE_FILE = { 'nx': 3, 'ny': 3, 'sx': 60, 'sy': 60, 'step_x': 1.5, 'step_y': 1.5} +AXES_MANAGER = { + 'axis-0': { + 'name': 'y', 'scale': 1.5, 'offset': 0.0, 'size': 3, 'units': 'μm', + 'navigate': True}, + 'axis-1': { + 'name': 'x', 'scale': 1.5, 'offset': 0.0, 'size': 3, 'units': 'μm', + 'navigate': True}, + 'axis-2': { + 'name': 'dy', 'scale': 1.0, 'offset': 0.0, 'size': 60, 'units': 'μm', + 'navigate': False}, + 'axis-3': { + 'name': 'dx', 'scale': 1.0, 'offset': 0.0, 'size': 60, 'units': 'μm', + 'navigate': False}} + @pytest.fixture() def save_path(): @@ -111,9 +136,123 @@ def save_path(): class TestNORDIF: def test_get_settings_from_file(self): - setting_file = os.path.join(NORDIF_PATH, 'Setting.txt') - settings = get_settings_from_file(setting_file) - answers = [METADATA, ORIGINAL_METADATA, AXES_MANAGER] + settings = get_settings_from_file(SETTING_FILE) + + answers = [METADATA, ORIGINAL_METADATA, SCAN_SIZE_FILE] assert len(settings) == len(answers) for setting_read, answer in zip(settings, answers): np.testing.assert_equal(setting_read.as_dictionary(), answer) + + @pytest.mark.parametrize('setting_file', (None, SETTING_FILE)) + def test_load(self, setting_file): + s = kp.load(PATTERN_FILE, setting_file=SETTING_FILE) + + assert s.data.shape == (3, 3, 60, 60) + assert s.axes_manager.as_dictionary() == AXES_MANAGER + + static_bg = plt.imread(BG_FILE) + assert s.metadata.Acquisition_instrument.SEM.Detector.EBSD\ + .static_background.all() == static_bg.all() + + @pytest.mark.parametrize('nav_shape, sig_shape', [ + ((3, 3), (60, 60)), ((3, 4), (50, 50))]) + def test_load_parameters(self, nav_shape, sig_shape): + if sum(nav_shape + sig_shape) > 126: + # Check if zero padding user warning is raised if sum of data shape + # is bigger than file size + with pytest.warns(UserWarning): + s = kp.load( + PATTERN_FILE, scan_size=nav_shape, pattern_size=sig_shape) + else: + s = kp.load( + PATTERN_FILE, scan_size=nav_shape, pattern_size=sig_shape) + + assert s.data.shape == nav_shape[::-1] + sig_shape + + def test_load_save_cycle(self, save_path): + s_reload = None + s = kp.load(PATTERN_FILE) + + scan_time_string = s.original_metadata['nordif_header'][80][10:18] + scan_time = time.strptime(scan_time_string, '%H:%M:%S') + scan_time = datetime.timedelta( + hours=scan_time.tm_hour, minutes=scan_time.tm_min, + seconds=scan_time.tm_sec).total_seconds() + assert s.metadata.Acquisition_instrument.SEM.Detector.EBSD.scan_time\ + == scan_time + assert s.metadata.General.title == 'Pattern' + + s.save(save_path, overwrite=True) + with pytest.warns(UserWarning): # No background pattern in directory + s_reload = kp.load(save_path, setting_file=SETTING_FILE) + np.testing.assert_equal(s.data, s_reload.data) + + # Add static background and change filename to make metadata equal + s_reload.metadata.Acquisition_instrument.SEM.Detector.EBSD\ + .static_background = plt.imread(BG_FILE) + s_reload.metadata.General\ + .original_filename = s.metadata.General.original_filename + s_reload.metadata.General.title = s.metadata.General.title + np.testing.assert_equal( + s_reload.metadata.as_dictionary(), s.metadata.as_dictionary()) + + # Delete reference to close np.memmap file + del s_reload + + def test_load_lazy(self): + s = kp.load(PATTERN_FILE, lazy=True) + assert isinstance(s.data, da.Array) + + def test_load_to_memory(self): + s = kp.load(PATTERN_FILE, lazy=False) + assert isinstance(s.data, np.ndarray) + assert not isinstance(s.data, np.memmap) + + def test_load_readonly(self): + s = kp.load(PATTERN_FILE, lazy=True) + k = next(filter(lambda x: isinstance(x, str) and + x.startswith("array-original"), + s.data.dask.keys())) + mm = s.data.dask[k] + assert isinstance(mm, np.memmap) + assert not mm.flags["WRITEABLE"] + with pytest.raises(NotImplementedError): + s.data[:] = 23 + + def test_load_inplace(self): + with pytest.raises(ValueError): + kp.load(PATTERN_FILE, lazy=True, mmap_mode='r+') + + def test_save_fresh(self, save_path): + scan_size = (10, 3) + pattern_size = (5, 5) + data_shape = scan_size + pattern_size + s = kp.signals.EBSD( + (255 * np.random.rand(*data_shape)).astype(np.uint8)) + s.save(save_path, overwrite=True) + with pytest.warns(UserWarning): # No background or setting files + s_reload = kp.load( + save_path, scan_size=scan_size[::-1], pattern_size=pattern_size) + np.testing.assert_equal(s.data, s_reload.data) + + def test_write_data_line(self, save_path): + scan_size = 3 + pattern_size = (5, 5) + data_shape = (scan_size, ) + pattern_size + s = kp.signals.EBSD( + (255 * np.random.rand(*data_shape)).astype(np.uint8)) + s.save(save_path, overwrite=True) + with pytest.warns(UserWarning): # No background or setting files + s_reload = kp.load( + save_path, scan_size=scan_size, pattern_size=pattern_size) + np.testing.assert_equal(s.data, s_reload.data) + + def test_write_data_single(self, save_path): + pattern_size = (5, 5) + s = kp.signals.EBSD( + (255 * np.random.rand(*pattern_size)).astype(np.uint8)) + s.save(save_path, overwrite=True) + with pytest.warns(UserWarning): # No background or setting files + s_reload = kp.load( + save_path, scan_size=1, pattern_size=pattern_size) + np.testing.assert_equal(s.data, s_reload.data) From 828aeae83ef63d7b8e8e0a43162a55bfa369e94a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Wiik=20=C3=85nes?= Date: Sun, 22 Sep 2019 12:59:49 +0200 Subject: [PATCH 3/5] Add padding test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Håkon Wiik Ånes --- kikuchipy/io_plugins/test/test_nordif.py | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/kikuchipy/io_plugins/test/test_nordif.py b/kikuchipy/io_plugins/test/test_nordif.py index ebf0378d..4bc0256b 100644 --- a/kikuchipy/io_plugins/test/test_nordif.py +++ b/kikuchipy/io_plugins/test/test_nordif.py @@ -26,6 +26,7 @@ import gc import time import datetime +import warnings import numpy as np import dask.array as da import matplotlib.pyplot as plt @@ -256,3 +257,28 @@ def test_write_data_single(self, save_path): s_reload = kp.load( save_path, scan_size=1, pattern_size=pattern_size) np.testing.assert_equal(s.data, s_reload.data) + + def test_read_cutoff(self, save_path): + scan_size = (10, 3) + scan_size_reloaded = (10, 20) + pattern_size = (5, 5) + data_shape = scan_size + pattern_size + s = kp.signals.EBSD( + (255 * np.random.rand(*data_shape)).astype(np.uint8)) + s.save(save_path, overwrite=True) + + # Reload data but with a scan_size bigger than available file bytes, + # so that the data has to be padded + with pytest.warns(UserWarning): # No background or setting files + s_reload = kp.load( + save_path, scan_size=scan_size_reloaded[::-1], + pattern_size=pattern_size) + + # To check if the data padding works as expected, the original data is + # padded and compared to the reloaded data + cut_data = s.data.flatten() + pw = [(0, (scan_size_reloaded[1] - scan_size[1]) * scan_size[0] + * np.prod(pattern_size))] + cut_data = np.pad(cut_data, pw, mode='constant') + cut_data = cut_data.reshape(scan_size_reloaded + pattern_size) + np.testing.assert_equal(cut_data, s_reload.data) From 96b64e62a1db56a739d0e33debb93e7fea743944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Wiik=20=C3=85nes?= Date: Sun, 22 Sep 2019 13:24:30 +0200 Subject: [PATCH 4/5] Add test for get_string in NORDIF setting file reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Håkon Wiik Ånes --- kikuchipy/io_plugins/test/test_nordif.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/kikuchipy/io_plugins/test/test_nordif.py b/kikuchipy/io_plugins/test/test_nordif.py index 4bc0256b..b528141d 100644 --- a/kikuchipy/io_plugins/test/test_nordif.py +++ b/kikuchipy/io_plugins/test/test_nordif.py @@ -31,7 +31,7 @@ import dask.array as da import matplotlib.pyplot as plt import kikuchipy as kp -from kikuchipy.io_plugins.nordif import get_settings_from_file +from kikuchipy.io_plugins.nordif import get_settings_from_file, get_string DIR_PATH = os.path.dirname(__file__) @@ -144,6 +144,21 @@ def test_get_settings_from_file(self): for setting_read, answer in zip(settings, answers): np.testing.assert_equal(setting_read.as_dictionary(), answer) + @pytest.mark.parametrize('line_no, correct', [(10, True), (11, False)]) + def test_get_string(self, line_no, correct): + f = open(SETTING_FILE, 'r', encoding='latin-1') + content = f.read().splitlines() + exp = 'Tilt angle\t(.*)\t' + if correct: + sample_tilt = get_string( + content=content, expression=exp, line_no=line_no, file=f) + assert sample_tilt == str(70) + else: + with pytest.warns(UserWarning): + sample_tilt = get_string( + content=content, expression=exp, line_no=line_no, file=f) + assert sample_tilt == 0 + @pytest.mark.parametrize('setting_file', (None, SETTING_FILE)) def test_load(self, setting_file): s = kp.load(PATTERN_FILE, setting_file=SETTING_FILE) @@ -171,7 +186,6 @@ def test_load_parameters(self, nav_shape, sig_shape): assert s.data.shape == nav_shape[::-1] + sig_shape def test_load_save_cycle(self, save_path): - s_reload = None s = kp.load(PATTERN_FILE) scan_time_string = s.original_metadata['nordif_header'][80][10:18] @@ -200,9 +214,13 @@ def test_load_save_cycle(self, save_path): # Delete reference to close np.memmap file del s_reload - def test_load_lazy(self): + def test_load_save_lazy(self, save_path): s = kp.load(PATTERN_FILE, lazy=True) assert isinstance(s.data, da.Array) + s.save(save_path, overwrite=True) + with pytest.warns(UserWarning): # No background pattern in directory + s_reload = kp.load(save_path, lazy=True, setting_file=SETTING_FILE) + assert s.data.shape == s_reload.data.shape def test_load_to_memory(self): s = kp.load(PATTERN_FILE, lazy=False) From c203be2f1e7157319a3d0b5136f4195da80d7deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Wiik=20=C3=85nes?= Date: Sun, 22 Sep 2019 15:06:57 +0200 Subject: [PATCH 5/5] Add tests for mmap_mode='r+b' and not lazy and padding check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Håkon Wiik Ånes --- kikuchipy/io_plugins/nordif.py | 5 ++-- kikuchipy/io_plugins/test/test_nordif.py | 37 +++++++++++++++--------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/kikuchipy/io_plugins/nordif.py b/kikuchipy/io_plugins/nordif.py index 517b9f2c..4f1c2465 100644 --- a/kikuchipy/io_plugins/nordif.py +++ b/kikuchipy/io_plugins/nordif.py @@ -75,10 +75,9 @@ def file_reader( 'write' in mmap_mode and 'copyonwrite' != mmap_mode): if lazy: raise ValueError("Lazy loading does not support in-place writing") - f = open(filename, 'r+b') - scan['attributes']['_lazy'] = True + f = open(filename, mode='r+b') else: - f = open(filename, 'rb') + f = open(filename, mode='rb') # Get metadata from setting file sem_node, ebsd_node = metadata_nodes() diff --git a/kikuchipy/io_plugins/test/test_nordif.py b/kikuchipy/io_plugins/test/test_nordif.py index b528141d..36812fa6 100644 --- a/kikuchipy/io_plugins/test/test_nordif.py +++ b/kikuchipy/io_plugins/test/test_nordif.py @@ -171,19 +171,25 @@ def test_load(self, setting_file): .static_background.all() == static_bg.all() @pytest.mark.parametrize('nav_shape, sig_shape', [ - ((3, 3), (60, 60)), ((3, 4), (50, 50))]) + ((3, 3), (60, 60)), ((3, 4), (60, 60)), (None, None)]) def test_load_parameters(self, nav_shape, sig_shape): - if sum(nav_shape + sig_shape) > 126: - # Check if zero padding user warning is raised if sum of data shape - # is bigger than file size - with pytest.warns(UserWarning): + if nav_shape is None and sig_shape is None: + with pytest.raises(ValueError): + kp.load( + PATTERN_FILE, setting_file='Setting.txt', + scan_size=nav_shape, pattern_size=sig_shape) + else: + if sum(nav_shape + sig_shape) > 126: + # Check if zero padding user warning is raised if sum of data + # shape is bigger than file size + with pytest.warns(UserWarning): + s = kp.load( + PATTERN_FILE, scan_size=nav_shape, + pattern_size=sig_shape) + else: s = kp.load( PATTERN_FILE, scan_size=nav_shape, pattern_size=sig_shape) - else: - s = kp.load( - PATTERN_FILE, scan_size=nav_shape, pattern_size=sig_shape) - - assert s.data.shape == nav_shape[::-1] + sig_shape + assert s.data.shape == nav_shape[::-1] + sig_shape def test_load_save_cycle(self, save_path): s = kp.load(PATTERN_FILE) @@ -238,9 +244,14 @@ def test_load_readonly(self): with pytest.raises(NotImplementedError): s.data[:] = 23 - def test_load_inplace(self): - with pytest.raises(ValueError): - kp.load(PATTERN_FILE, lazy=True, mmap_mode='r+') + @pytest.mark.parametrize('lazy', [True, False]) + def test_load_inplace(self, lazy): + if lazy: + with pytest.raises(ValueError): + s = kp.load(PATTERN_FILE, lazy=lazy, mmap_mode='r+') + else: + s = kp.load(PATTERN_FILE, lazy=lazy, mmap_mode='r+') + assert s.axes_manager.as_dictionary() == AXES_MANAGER def test_save_fresh(self, save_path): scan_size = (10, 3)