Merge pull request #371 from hakonanes/oxford-binary-format-reader

Add reader of EBSD patterns from Oxford binary .ebsp files

doc/changelog.rst

@@ -21,6 +21,9 @@ Contributors
 
 Added
 -----
+- Reader for EBSD patterns stored in Oxford Instrument's binary .ebsp file
+  format, uncompressed and with intensities as 8-bit unsigned integers.
+  (`#? <https://github.com/pyxem/kikuchipy/pull/?>`_)
 - Unit testing of docstring examples.
   (`#350 <https://github.com/pyxem/kikuchipy/pull/350>`_)
 - Support for Python 3.9.

doc/load_save_data.ipynb

@@ -507,11 +507,12 @@
     "| ------------------------------- | ---- | ----- |\n",
     "| Bruker Nano h5ebsd              | Yes  | No    |\n",
     "| EDAX TSL h5ebsd                 | Yes  | No    |\n",
+    "| EMsoft simulated EBSD HDF5      | Yes  | No    |\n",
+    "| EMsoft EBSD master pattern HDF5 | Yes  | No    |\n",
     "| kikuchipy h5ebsd                | Yes  | Yes   |\n",
     "| NORDIF binary                   | Yes  | Yes   |\n",
     "| NORDIF calibration patterns     | Yes  | No    |\n",
-    "| EMsoft simulated EBSD HDF5      | Yes  | No    |\n",
-    "| EMsoft EBSD master pattern HDF5 | Yes  | No    |\n",
+    "| Oxford Instruments binary       | Yes  | No    |\n",
     "\n",
     "<div class=\"alert alert-info\">\n",
     "\n",
@@ -552,7 +553,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "kikuchipy_ebsd = \"kikuchipy/patterns.h5\"\n",
+    "kikuchipy_ebsd = \"kikuchipy_h5ebsd/patterns.h5\"\n",
     "s_awsm, s2 = kp.load(\n",
     "    filename=datadir + kikuchipy_ebsd,\n",
     "    scan_group_names=[\"My awes0m4 Xcan #! with a long title\", \"Scan 2\"]\n",
@@ -853,6 +854,53 @@
     "details of the underlying theory."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Oxford Instrument's binary\n",
+    "\n",
+    "Patterns stored in the Oxford Instruments binary .ebsp file format, uncompressed\n",
+    "and with intensities as 8-bit unsigned integer, can be read by providing the\n",
+    "navigation shape (number of map rows, number of map columns), which is not\n",
+    "stored explicitly in the file"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "oxford_binary_path = datadir + \"oxford_binary/\"\n",
+    "s_oxford = kp.load(filename=oxford_binary_path + \"patterns.ebsp\",\n",
+    "    navigation_shape=(3, 3),\n",
+    ")\n",
+    "s_oxford"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Here, the Oxford Instruments binary\n",
+    "[file_reader()](reference.rst#kikuchipy.io.plugins.oxford_binary.file_reader) is\n",
+    "called. If the navigation shape is not known, the number of patterns in the file\n",
+    "can be guessed by the reader from inspecting the file header. The returned\n",
+    "signal will then have a single navigation dimension the size of the number of\n",
+    "patterns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "s_oxford2 = kp.load(filename=oxford_binary_path + \"patterns.ebsp\")\n",
+    "s_oxford2"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

doc/reference.rst

@@ -308,6 +308,7 @@ plugins
     nordif_calibration_patterns
     emsoft_ebsd
     emsoft_ebsd_master_pattern
+    oxford_binary
 
 The plugins import patterns and parameters from file formats into
 :class:`~kikuchipy.signals.EBSD` or
@@ -355,6 +356,14 @@ emsoft_ebsd_master_pattern
     :undoc-members:
     :show-inheritance:
 
+oxford_binary
+~~~~~~~~~~~~~
+
+.. automodule:: kikuchipy.io.plugins.oxford_binary
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 ....
 
 pattern

kikuchipy/conftest.py

@@ -409,4 +409,4 @@ def doctest_setup_teardown(request):
 @pytest.fixture(autouse=True)
 def import_to_namespace(doctest_namespace):
     DIR_PATH = os.path.dirname(__file__)
-    doctest_namespace["DATA_DIR"] = os.path.join(DIR_PATH, "data/kikuchipy")
+    doctest_namespace["DATA_DIR"] = os.path.join(DIR_PATH, "data/kikuchipy_h5ebsd")

kikuchipy/data/__init__.py

@@ -22,8 +22,8 @@
 see :doc:`open datasets <open_datasets>`.
 """
 
-from pathlib import Path
 import os
+from pathlib import Path
 from typing import Union
 
 import pooch as ppooch
@@ -111,7 +111,7 @@ def nickel_ebsd_small(**kwargs) -> EBSD:
     signal : EBSD
         EBSD signal.
     """
-    return _load(filename="data/kikuchipy/patterns.h5", **kwargs)
+    return _load(filename="data/kikuchipy_h5ebsd/patterns.h5", **kwargs)
 
 
 def nickel_ebsd_master_pattern_small(**kwargs) -> EBSDMasterPattern:

kikuchipy/data/_registry.py

@@ -18,7 +18,7 @@
 
 # fmt: off
 registry = {
-    "data/kikuchipy/patterns.h5": "7a99ce88174c725f5407b6fcc1eab0c4255694ca8e6029fde7f372f3ab40897f",
+    "data/kikuchipy_h5ebsd/patterns.h5": "7a99ce88174c725f5407b6fcc1eab0c4255694ca8e6029fde7f372f3ab40897f",
     "data/emsoft_ebsd_master_pattern/ni_mc_mp_20kv_uint8_gzip_opts9.h5": "8a7c1fb471d9ce750f0332a154e87cf41eed7529be508548e0c0f51ec6f92bc2",
     "data/nickel_ebsd_large/patterns.h5": "3ea6e729c3adfdea9dce461806f011c24bf70b011dcf4d90a23a6aa29f15872c",
     "data/silicon_ebsd_moving_screen/si_in.h5": "de57b8ef7213420af694c191c939f44818979b3db9873e74cc85cabde824b7eb",

kikuchipy/data/oxford_binary/create_oxford_binary_file.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019-2021 The kikuchipy developers
+#
+# This file is part of kikuchipy.
+#
+# kikuchipy is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# kikuchipy is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with kikuchipy. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+import numpy as np
+
+import kikuchipy as kp
+
+
+s = kp.data.nickel_ebsd_small()
+
+sr, sc = s.axes_manager.signal_shape[::-1]
+n_pixels = sr * sc
+nr, nc = s.axes_manager.navigation_shape[::-1]
+n_patterns = nr * nc
+
+dir_data = os.path.abspath(os.path.dirname(__file__))
+fname = os.path.join(dir_data, "patterns.ebsp")
+file = open(fname, mode="w")
+
+# Write file header: 8 bytes with ?
+file_header = np.ones(1, dtype=np.int64)
+file_header.tofile(file)
+
+# Pattern header: 16 bytes with pattern height, pattern width and ?
+pattern_header_size = 16
+pattern_header = np.zeros(8, dtype=np.uint16)
+pattern_header[2] = sr
+pattern_header[4] = sc
+
+# Pattern footer: 18 bytes with ?
+pattern_footer_size = 18
+pattern_footer = np.zeros(9, dtype=np.uint16)
+
+# Write pattern positions: 8 bytes per position
+pattern_positions = np.arange(n_patterns, dtype=np.int64)
+pattern_positions *= pattern_header_size + n_pixels + pattern_footer_size
+pattern_positions += file_header.nbytes + n_patterns * 8
+# Shift positions one step to the right
+new_order = np.roll(np.arange(n_patterns), 1)
+pattern_positions = pattern_positions[new_order]
+pattern_positions.tofile(file)
+
+# Write patterns with header and footer
+pattern_data = s.data
+for i in new_order:
+    r, c = np.unravel_index(i, (nr, nc))
+    pattern_header.tofile(file)
+    pattern_data[r, c].tofile(file)
+    pattern_footer.tofile(file)
+
+file.close()

kikuchipy/generators/tests/test_virtual_bse_generator.py

@@ -29,7 +29,7 @@
 from kikuchipy.signals import EBSD, LazyEBSD, VirtualBSEImage
 
 DIR_PATH = os.path.dirname(__file__)
-KIKUCHIPY_FILE = os.path.join(DIR_PATH, "../../data/kikuchipy/patterns.h5")
+KIKUCHIPY_FILE = os.path.join(DIR_PATH, "../../data/kikuchipy_h5ebsd/patterns.h5")
 
 matplotlib.use("Agg")  # For plotting
 

kikuchipy/io/_io.py

@@ -33,6 +33,7 @@
     h5ebsd,
     nordif,
     nordif_calibration_patterns,
+    oxford_binary,
 )
 from kikuchipy.io._util import _get_input_bool, _ensure_directory
 
@@ -44,6 +45,7 @@
     h5ebsd,
     nordif,
     nordif_calibration_patterns,
+    oxford_binary,
 ]
 
 default_write_ext = set()

kikuchipy/io/plugins/__init__.py

@@ -23,11 +23,13 @@
     emsoft_ebsd_master_pattern,
     h5ebsd,
     nordif,
+    oxford_binary,
 )
 
 __all__ = [
     "emsoft_ebsd",
     "emsoft_ebsd_master_pattern",
     "h5ebsd",
     "nordif",
+    "oxford_binary",
 ]

kikuchipy/io/plugins/nordif_calibration_patterns.py

@@ -118,7 +118,7 @@ def _get_coordinates(filename: str) -> List[Tuple[int, int]]:
         raise ValueError(err)
     xy = []
     for line in content[l_start + 1 :]:
-        match = re.search("Calibration \((.*)\)", line)
+        match = re.search(r"Calibration \((.*)\)", line)
         try:
             match = match.group(1)
             match = match.split(",")