Merge 0398451 into e342a2a

.travis.yml

@@ -33,8 +33,8 @@ before_install:
 install:
 
   - if [[ $MINIMAL_ENV == 'False' ]] ; then
-      DEPS="pip numpy scipy matplotlib ipython h5py sympy scikit-learn dill natsort setuptools scikit-image cython ipyparallel dask traits traitsui numexpr";
-    else DEPS="pip ipython numpy scipy matplotlib h5py sympy scikit-image numexpr";
+      DEPS="pip numpy scipy matplotlib=2.2.3 ipython h5py sympy scikit-learn dill natsort setuptools scikit-image cython ipyparallel dask traits traitsui numexpr numba";
+    else DEPS="pip ipython numpy scipy matplotlib=2.2.3 h5py sympy scikit-image numexpr";
     fi
   - conda create -n testenv --yes python=$PYTHON;
     source activate testenv;

appveyor.yml

@@ -4,7 +4,7 @@ shallow_clone: true
 environment:
 
   global:
-    TEST_DEPS: "pytest pytest-cov wheel"
+    TEST_DEPS: "pytest pytest-cov pytest-mpl wheel"
     MPLBACKEND: "agg"
 
 
@@ -17,28 +17,28 @@ environment:
        PYTHON_MAJOR: 3
        PYTHON_ARCH: "32"
        CONDA_PY: "35"
-       DEPS: "numpy scipy matplotlib ipython h5py sympy scikit-learn dill setuptools natsort scikit-image cython ipyparallel dask numexpr"
+       DEPS: "numpy scipy matplotlib=2.2.3 ipython h5py sympy scikit-learn dill setuptools natsort scikit-image cython ipyparallel dask numexpr sparse numba"
 
      - PYTHON: "C:\\Miniconda35-x64"
        PYTHON_VERSION: "3.5.x"
        PYTHON_MAJOR: 3
        PYTHON_ARCH: "64"
        CONDA_PY: "35"
-       DEPS: "numpy scipy matplotlib ipython h5py sympy scikit-learn dill setuptools natsort scikit-image cython ipyparallel dask numexpr"
+       DEPS: "numpy scipy matplotlib=2.2.3 ipython h5py sympy scikit-learn dill setuptools natsort scikit-image cython ipyparallel dask numexpr sparse numba"
 
      - PYTHON: "C:\\Miniconda36"
        PYTHON_VERSION: "3.6.x"
        PYTHON_MAJOR: 3
        PYTHON_ARCH: "32"
        CONDA_PY: "36"
-       DEPS: "numpy scipy matplotlib ipython h5py sympy scikit-learn dill setuptools natsort scikit-image cython ipyparallel dask numexpr"
+       DEPS: "numpy scipy matplotlib=2.2.3 ipython h5py sympy scikit-learn dill setuptools natsort scikit-image cython ipyparallel dask numexpr sparse numba"
 
      - PYTHON: "C:\\Miniconda36-x64"
        PYTHON_VERSION: "3.6.x"
        PYTHON_MAJOR: 3
        PYTHON_ARCH: "64"
        CONDA_PY: "36"
-       DEPS: "numpy scipy matplotlib ipython h5py sympy scikit-learn dill setuptools natsort scikit-image cython ipyparallel dask numexpr"
+       DEPS: "numpy scipy matplotlib=2.2.3 ipython h5py sympy scikit-learn dill setuptools natsort scikit-image cython ipyparallel dask numexpr sparse numba"
 
 
 
@@ -60,15 +60,11 @@ install:
 
   # Install the dependencies of the project.
   - ps: Add-AppveyorMessage "Installing conda packages..."
-  - "%CMD_IN_ENV% conda install -yq %TEST_DEPS%"
-  - "%CMD_IN_ENV% conda install -yq %DEPS%"
-  - "pip install pytest-mpl"
-
-  # Having 'sip' folder on path confuses import of `sip`.
-  #- "%CMD_IN_ENV% conda install -yq pip"
+  - "conda install -yq -c conda-forge %TEST_DEPS%"
+  - "conda install -yq -c conda-forge %DEPS%"
 
   - ps: Add-AppveyorMessage "Installing hyperspy..."
-  - "python setup.py install"
+  - "pip install .[all]"
 
 build: false  # Not a C# project, build stuff at the test step instead.
 

hyperspy/io_plugins/protochips.py

@@ -67,17 +67,18 @@ def _protochips_log_reader(csv_file):
 
 class ProtochipsCSV(object):
 
-    def __init__(self, filename, header_line_number=10):
+    def __init__(self, filename, ):
         self.filename = filename
-        self._parse_header(header_line_number)
-        self._read_data(header_line_number)
+        self._parse_header()
+        self._read_data()
 
-    def _parse_header(self, header_line_number):
-        self.raw_header = self._read_header(header_line_number)
-        self.column_name = self._read_column_name()
-        if not self._is_protochips_csv_file():
-            raise IOError(invalid_file_error)
-        self._read_all_metadata_header()
+    def _parse_header(self):
+        with open(self.filename, 'r') as f:
+            s = f.readline()
+            self.column_name = s.replace(', ', ',').replace('\n', '').split(',')
+            if not self._is_protochips_csv_file():
+                raise IOError(invalid_file_error)
+            self._read_all_metadata_header(f)
         self.logged_quantity_name_list = self.column_name[2:]
 
     def _is_protochips_csv_file(self):
@@ -135,14 +136,14 @@ def _get_metadata_time_axis(self):
         return {'value': self.time_axis,
                 'units': self.time_units}
 
-    def _read_data(self, header_line_number):
+    def _read_data(self):
         names = [name.replace(' ', '_') for name in self.column_name]
         # Necessary for numpy >= 1.14
         kwargs = {'encoding': 'latin1'} if np.__version__ >= LooseVersion("1.14") else {
         }
         data = np.genfromtxt(self.filename, delimiter=',', dtype=None,
                              names=names,
-                             skip_header=header_line_number,
+                             skip_header=self.header_last_line_number,
                              unpack=True, **kwargs)
 
         self._data_dictionary = dict()
@@ -173,8 +174,8 @@ def _parse_calibration_file_name(self, path):
         return "Calibration file name: %s" % basename.split('\\')[-1]
 
     def _get_axes(self):
-        scale = np.diff(self.time_axis[1:-2]).mean()
-        max_diff = np.diff(self.time_axis[1:-2]).max()
+        scale = np.diff(self.time_axis[1:-1]).mean()
+        max_diff = np.diff(self.time_axis[1:-1]).max()
         units = 's'
         offset = 0
         if self.time_units == 'Milliseconds':
@@ -206,14 +207,9 @@ def _parse_quantity_units(self, quantity):
         quantity = quantity.split(' ')[-1].lower()
         return self.__dict__['%s_units' % quantity]
 
-    def _read_header(self, header_line_number):
-        with open(self.filename, 'r') as f:
-            raw_header = [f.readline() for i in range(header_line_number)]
-        return raw_header
-
-    def _read_all_metadata_header(self):
-        i = 1
-        param, value = self._parse_metadata_header(self.raw_header[i])
+    def _read_all_metadata_header(self, f):
+        param, value = self._parse_metadata_header(f.readline())
+        i = 2
         while 'User' not in param:  # user should be the last of the header
             if 'Calibration file' in param:
                 self.calibration_file = value
@@ -226,21 +222,20 @@ def _read_all_metadata_header(self):
                 self.__dict__[attr_name] = value
             i += 1
             try:
-                param, value = self._parse_metadata_header(self.raw_header[i])
-            except ValueError:  # when the last line of header does not contain 'User'
+                param, value = self._parse_metadata_header(f.readline())
+            except ValueError:
+                # when the last line of header does not contain 'User',
+                # possibly some old file.
                 self.user = None
                 break
             except IndexError:
                 _logger.warning("The metadata may not be parsed properly.")
                 break
         else:
             self.user = value
+        self.header_last_line_number = i
         self.start_datetime = np.datetime64(dt.strptime(date + time,
                                                         "%Y.%m.%d%H:%M:%S.%f"))
 
     def _parse_metadata_header(self, line):
         return line.replace(', ', ',').split(',')[1].split(' = ')
-
-    def _read_column_name(self):
-        string = self.raw_header[0]
-        return string.replace(', ', ',').replace('\n', '').split(',')

hyperspy/misc/io/fei_stream_readers.py

@@ -1,25 +1,20 @@
 import numpy as np
 import dask.array as da
+import sparse
 
 from hyperspy.decorators import jit_ifnumba
 
 
-try:
-    import sparse
-    sparse_installed = True
+class DenseSliceCOO(sparse.COO):
+    """Just like sparse.COO, but returning a dense array on indexing/slicing"""
 
-    class DenseSliceCOO(sparse.COO):
-        """Just like sparse.COO, but returning a dense array on indexing/slicing"""
-
-        def __getitem__(self, *args, **kwargs):
-            obj = super().__getitem__(*args, **kwargs)
-            try:
-                return obj.todense()
-            except AttributeError:
-                # Indexing, unlike slicing, returns directly the content
-                return obj
-except ImportError:
-    sparse_installed = False
+    def __getitem__(self, *args, **kwargs):
+        obj = super().__getitem__(*args, **kwargs)
+        try:
+            return obj.todense()
+        except AttributeError:
+            # Indexing, unlike slicing, returns directly the content
+            return obj
 
 
 @jit_ifnumba()
@@ -29,8 +24,11 @@ def _stream_to_sparse_COO_array_sum_frames(
     frame_number = 0
     ysize, xsize = shape
     frame_size = xsize * ysize
-    data_list = []
-    coords_list = []
+    # workaround for empty stream, numba "doesn't support" empty list, see
+    # https://github.com/numba/numba/pull/2184
+    # add first element and remove it at the end
+    data_list = [0]
+    coords_list = [(0, 0, 0)]
     data = 0
     count_channel = None
     for value in stream_data:
@@ -94,8 +92,9 @@ def _stream_to_sparse_COO_array_sum_frames(
         data_list.append(data)
 
     final_shape = (ysize, xsize, channels // rebin_energy)
-    coords = np.array(coords_list).T
-    data = np.array(data_list)
+    # Remove first element, see comments above
+    coords = np.array(coords_list)[1:].T
+    data = np.array(data_list)[1:]
     return coords, data, final_shape
 
 
@@ -106,8 +105,11 @@ def _stream_to_sparse_COO_array(
     frame_number = 0
     ysize, xsize = shape
     frame_size = xsize * ysize
-    data_list = []
-    coords = []
+    # workaround for empty stream, numba "doesn't support" empty list, see
+    # https://github.com/numba/numba/pull/2184
+    # add first element and remove it at the end
+    data_list = [0]
+    coords = [(0, 0, 0, 0)]
     data = 0
     count_channel = None
     for value in stream_data:
@@ -175,8 +177,9 @@ def _stream_to_sparse_COO_array(
 
     final_shape = (last_frame - first_frame, ysize, xsize,
                    channels // rebin_energy)
-    coords = np.array(coords).T
-    data = np.array(data_list)
+    # Remove first element, see comments above
+    coords = np.array(coords)[1:].T
+    data = np.array(data_list)[1:]
     return coords, data, final_shape
 
 
@@ -198,11 +201,6 @@ def stream_to_sparse_COO_array(
         If True, sum all the frames
 
     """
-    if not sparse_installed:
-        raise ImportError(
-            "The python-sparse package is not installed and it is required "
-            "for lazy loading of SIs stored in FEI EMD stream format."
-        )
     if sum_frames:
         coords, data, shape = _stream_to_sparse_COO_array_sum_frames(
             stream_data=stream_data,

hyperspy/tests/io/protochips_data/invalid_protochips_file.csv

@@ -12,4 +12,3 @@ Time, Notes, Channel A Current, Channel A Voltage, Channel A Resistance, Channel
 196, , 0.0179984, 39.9278, 2218.40830296026, 0, 0, NaN
 296, , 0.0179992, 39.7102, 2206.22027645673, 0, 0, NaN
 396, , Test, 39.5232, 2195.79432762021, 0, 0, NaN
-

hyperspy/tests/io/test_emd.py

@@ -37,7 +37,6 @@
 from hyperspy.io import load
 from hyperspy.signals import BaseSignal, Signal2D, Signal1D, EDSTEMSpectrum
 from hyperspy.misc.test_utils import assert_warns, assert_deep_almost_equal
-from hyperspy.misc.io.fei_stream_readers import sparse_installed
 
 
 my_path = os.path.dirname(__file__)
@@ -360,8 +359,6 @@ def test_fei_emd_spectrum(self, lazy):
 
     @pytest.mark.parametrize("lazy", (True, False))
     def test_fei_emd_si(self, lazy):
-        if lazy and not sparse_installed:
-            pytest.skip("python sparse is not installed")
         signal = load(os.path.join(self.fei_files_path, 'fei_emd_si.emd'),
                       lazy=lazy)
         if lazy:
@@ -373,8 +370,6 @@ def test_fei_emd_si(self, lazy):
 
     @pytest.mark.parametrize("lazy", (True, False))
     def test_fei_emd_si_non_square_10frames(self, lazy):
-        if lazy and not sparse_installed:
-            pytest.skip("python sparse is not installed")
         s = load(os.path.join(
             self.fei_files_path, 'fei_SI_SuperX-HAADF_10frames_10x50.emd'),
             lazy=lazy)
@@ -500,8 +495,6 @@ def test_fei_emd_si_non_square_10frames(self, lazy):
 
     @pytest.mark.parametrize("lazy", (True, False))
     def test_fei_emd_si_non_square_20frames(self, lazy):
-        if lazy and not sparse_installed:
-            pytest.skip("python sparse is not installed")
         s = load(os.path.join(
             self.fei_files_path,
             'fei_SI_SuperX-HAADF_20frames_10x50.emd'),
@@ -526,8 +519,6 @@ def test_fei_emd_si_non_square_20frames(self, lazy):
 
     @pytest.mark.parametrize("lazy", (True, False))
     def test_fei_emd_si_non_square_20frames_2eV(self, lazy):
-        if lazy and not sparse_installed:
-            pytest.skip("python sparse is not installed")
         s = load(os.path.join(
             self.fei_files_path,
             'fei_SI_SuperX-HAADF_20frames_10x50_2ev.emd'),
@@ -552,8 +543,6 @@ def test_fei_emd_si_non_square_20frames_2eV(self, lazy):
 
     @pytest.mark.parametrize("lazy", (True, False))
     def test_fei_emd_si_frame_range(self, lazy):
-        if lazy and not sparse_installed:
-            pytest.skip("python sparse is not installed")
         signal = load(os.path.join(self.fei_files_path, 'fei_emd_si.emd'),
                       first_frame=2, last_frame=4, lazy=lazy)
         fei_si = np.load(os.path.join(self.fei_files_path,
@@ -567,8 +556,6 @@ def test_fei_emd_si_frame_range(self, lazy):
     @pytest.mark.parametrize(["lazy", "sum_EDS_detectors"],
                              _generate_parameters())
     def test_fei_si_4detectors(self, lazy, sum_EDS_detectors):
-        if lazy and not sparse_installed:
-            pytest.skip("python sparse is not installed")
         fname = os.path.join(self.fei_files_path,
                              'fei_SI_EDS-HAADF-4detectors_2frames.emd')
         signal = load(fname, sum_EDS_detectors=sum_EDS_detectors, lazy=lazy)

hyperspy/tests/misc/test_fei_stream_readers.py

@@ -11,16 +11,14 @@
 import pytest
 
 from hyperspy.misc.io.fei_stream_readers import (
-    array_to_stream, stream_to_array, stream_to_sparse_COO_array, sparse_installed)
+    array_to_stream, stream_to_array, stream_to_sparse_COO_array)
 
 
 @pytest.mark.parametrize("lazy", (True, False))
 def test_dense_stream(lazy):
     arr = np.random.randint(0, 65535, size=(2, 3, 4, 5)).astype("uint16")
     stream = array_to_stream(arr)
     if lazy:
-        if not sparse_installed:
-            pytest.skip("The sparse package is not installed")
         arrs = da.from_array(stream_to_sparse_COO_array(
             stream, spatial_shape=(3, 4), sum_frames=False, channels=5,
             last_frame=2), chunks=(1, 1, 2, 5))
@@ -38,8 +36,6 @@ def test_empty_stream(lazy):
     arr = np.zeros((2, 3, 4, 5), dtype="uint16")
     stream = array_to_stream(arr)
     if lazy:
-        if not sparse_installed:
-            pytest.skip("The sparse package is not installed")
         arrs = da.from_array(stream_to_sparse_COO_array(
             stream, spatial_shape=(3, 4), sum_frames=False, channels=5,
             last_frame=2), chunks=(1, 1, 2, 5))
@@ -60,8 +56,6 @@ def test_sparse_stream(lazy):
     arr[1, 1, 3, 3] = 3
     stream = array_to_stream(arr)
     if lazy:
-        if not sparse_installed:
-            pytest.skip("The sparse package is not installed")
         arrs = da.from_array(stream_to_sparse_COO_array(
             stream, spatial_shape=(3, 4), sum_frames=False, channels=5,
             last_frame=2), chunks=(1, 1, 2, 5))