Merge remote-tracking branch 'upstream/RELEASE_next_patch' into RELEA…

…SE_next_minor # Conflicts: # CHANGES.rst
hyperspy · Jun 3, 2021 · 384b4e6 · 384b4e6
2 parents eb093a4 + e65d604
commit 384b4e6
Show file tree

Hide file tree

Showing 34 changed files with 165 additions and 86 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -17,6 +17,7 @@ A few sentences and/or a bulleted list to describe and motivate the change:
 - [ ] update docstring (if appropriate),
 - [ ] update user guide (if appropriate),
 - [ ] add an changelog entry in the `upcoming_changes` folder (see [`upcoming_changes/README.rst`](https://github.com/hyperspy/hyperspy/blob/RELEASE_next_minor/upcoming_changes/README.rst)),
+- [ ] Check formatting changelog entry in the `readthedocs` doc build of this PR (link in github checks)
 - [ ] add tests,
 - [ ] ready for review.
 

diff --git a/CHANGES.rst b/CHANGES.rst
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -21,36 +21,45 @@ resources:
       # For more details on service connection endpoint, see
       # https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints
       endpoint: hyperspy # Azure DevOps service connection
+      ref: use_mamba
 
 strategy:
   matrix:
     Linux_Python38:
       vmImage: 'ubuntu-latest'
       PYTHON_VERSION: '3.8'
+      MAMBAFORGE_PATH: $(Agent.BuildDirectory)/mambaforge
     Linux_Python37:
       vmImage: 'ubuntu-latest'
       PYTHON_VERSION: '3.7'
+      MAMBAFORGE_PATH: $(Agent.BuildDirectory)/mambaforge
     Linux_Python36:
       vmImage: 'ubuntu-latest'
       PYTHON_VERSION: '3.6'
+      MAMBAFORGE_PATH: $(Agent.BuildDirectory)/mambaforge
     MacOS_Python38:
       vmImage: 'macOS-latest'
       PYTHON_VERSION: '3.8'
+      MAMBAFORGE_PATH: $(Agent.BuildDirectory)/mambaforge
     MacOS_Python37:
       vmImage: 'macOS-latest'
       PYTHON_VERSION: '3.7'
+      MAMBAFORGE_PATH: $(Agent.BuildDirectory)/mambaforge
     Windows_Python38:
       vmImage: 'windows-latest'
       PYTHON_VERSION: '3.8'
+      MAMBAFORGE_PATH: $(Agent.BuildDirectory)\mambaforge
     Windows_Python36:
       vmImage: 'windows-latest'
       PYTHON_VERSION: '3.6'
+      MAMBAFORGE_PATH: $(Agent.BuildDirectory)\mambaforge
 
 pool:
   vmImage: '$(vmImage)'
 
 steps:
 - template: azure_pipelines/clone_ci-scripts_repo.yml@templates
+- template: azure_pipelines/install_mambaforge.yml@templates
 - template: azure_pipelines/activate_conda.yml@templates
 - template: azure_pipelines/setup_anaconda_packages.yml@templates
 

diff --git a/conda_environment.yml b/conda_environment.yml
@@ -1,6 +1,5 @@
 name: test_env
 channels:
-- defaults
 - conda-forge
 dependencies:
 - dask-core >2.0
@@ -17,7 +16,6 @@ dependencies:
 - numpy
 - pint
 - prettytable
-- python
 - python-dateutil
 - pyyaml
 - requests

diff --git a/conda_environment_dev.yml b/conda_environment_dev.yml
@@ -1,6 +1,5 @@
 name: test_env
 channels:
-- defaults
 - conda-forge
 dependencies:
 # We pin freetype and matplotlib for the image comparison

diff --git a/doc/conf.py b/doc/conf.py
@@ -38,6 +38,7 @@
     'sphinx.ext.graphviz',
     'sphinx.ext.autosummary',
     'sphinx_toggleprompt',
+    'sphinxcontrib.towncrier',
 ]
 
 try:
@@ -226,7 +227,6 @@
 # If false, no module index is generated.
 #latex_domain_indices = True
 
-
 # -- Options for manual page output --------------------------------------
 
 # One entry per manual page. List of tuples
@@ -236,6 +236,15 @@
      ['The HyperSpy developers'], 1)
 ]
 
+
+# -- Options for towncrier_draft extension -----------------------------------
+
+# Options: draft/sphinx-version/sphinx-release
+towncrier_draft_autoversion_mode = 'draft'
+towncrier_draft_include_empty = False
+towncrier_draft_working_directory = ".."
+
+
 # Add the hyperspy website to the intersphinx domains
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
                        'hyperspyweb': ('https://hyperspy.org/', None),

diff --git a/doc/dev_guide/testing.rst b/doc/dev_guide/testing.rst
@@ -177,9 +177,8 @@ The testing matrix is as follows:
   See ``.github/workflows/tests.yml`` in the HyperSpy repository for further details.
 - **Azure Pipeline**: test a range of Python versions on Linux, MacOS and Windows;
   all dependencies are installed from `Anaconda Cloud <https://anaconda.org/>`_
-  using the `Anaconda "defaults" <https://anaconda.org/anaconda>`_ and the
-  `"conda-forge" <https://anaconda.org/conda-forge>`_ channel (in this order of
-  priority). See ``azure-pipelines.yml`` in the HyperSpy repository for further details.
+  using the `"conda-forge" <https://anaconda.org/conda-forge>`_ channel.
+  See ``azure-pipelines.yml`` in the HyperSpy repository for further details.
 
 This testing matrix has been designed to be simple and easy to maintain, whilst
 ensuring that packages from PyPI and Anaconda cloud are not mixed in order to

diff --git a/doc/index.rst b/doc/index.rst
@@ -13,7 +13,7 @@ Welcome to HyperSpy's documentation!
    :caption: User Guide
 
    user_guide/intro.rst
-   user_guide/what_is_new.rst
+   user_guide/changes.rst
    user_guide/install.rst
    user_guide/getting_started.rst
    user_guide/signal.rst

diff --git a/doc/user_guide/what_is_new.rst → doc/user_guide/changes.rst b/doc/user_guide/what_is_new.rst → doc/user_guide/changes.rst
diff --git a/doc/user_guide/install.rst b/doc/user_guide/install.rst
@@ -179,7 +179,7 @@ dependencies required by specific functionalities:
 * ``tests`` to install required libraries to run HyperSpy's unit tests,
 * ``build-doc`` to install required libraries to build HyperSpy's documentation,
 * ``dev`` to install all the above,
-* ``all`` to install all the above expect the development requirements
+* ``all`` to install all the above except the development requirements
   (``tests``, ``build-doc`` and ``dev``).
 
 For example:

diff --git a/hyperspy/io_plugins/empad.py b/hyperspy/io_plugins/empad.py
@@ -74,13 +74,18 @@ def _parse_xml(filename):
             'height':128,
             'raw_height':130,
             'record-by':'image'}
-    if om.has_item('root.count'):
+    if om.has_item('root.scan_parameters.series_count'):
         # Stack of images
         info.update({'series_count':int(om.root.scan_parameters.series_count)})
     elif om.has_item('root.pix_x') and om.has_item('root.pix_y'):
         # 2D x 2D
         info.update({'scan_x':int(om.root.pix_x),
                      'scan_y':int(om.root.pix_y)})
+    # in case root.pix_x and root.pix_y are not available
+    elif (om.has_item('root.scan_parameters.scan_resolution_x') and 
+              om.has_item('root.scan_parameters.scan_resolution_y')):
+        info.update({'scan_x':int(om.root.scan_parameters.scan_resolution_x),
+                     'scan_y':int(om.root.scan_parameters.scan_resolution_y)})
     else:
         raise IOError("Unsupported Empad file: the scan parameters cannot "
                       "be imported.")

diff --git a/hyperspy/io_plugins/hspy.py b/hyperspy/io_plugins/hspy.py
@@ -129,6 +129,11 @@ def file_reader(filename, backing_store=False,
         Load image lazily using dask
     **kwds, optional
     """
+    try:
+        # in case blosc compression is used
+        import hdf5plugin
+    except ImportError:
+        pass
     mode = kwds.pop('mode', 'r')
     f = h5py.File(filename, mode=mode, **kwds)
     # Getting the format version here also checks if it is a valid HSpy

diff --git a/hyperspy/misc/utils.py b/hyperspy/misc/utils.py
@@ -1234,10 +1234,11 @@ def process_function_blockwise(data,
             output_array[islice] = function(data[islice],
                                             **iter_dict,
                                             **kwargs)
-    try:
-        output_array = output_array.squeeze(-1)
-    except ValueError:
-        pass
+    if not (chunk_nav_shape == output_array.shape):
+        try:
+            output_array = output_array.squeeze(-1)
+        except ValueError:
+            pass
     return output_array
 
 

diff --git a/hyperspy/tests/io/empad_data/map128x128_version1.2.0.xml b/hyperspy/tests/io/empad_data/map128x128_version1.2.0.xml
@@ -0,0 +1 @@
+<root name="21"><timestamp timestamp="1621298392.213645" isoformat="2021-05-18T08:39:52.213645"/><software_version>1.2.0  (2020-10-29)</software_version><reconstructions Sentinel="Test  "><reconstruction absolute_filename="/home/empad/empad_projects/STO/21/scan_roi3_ud_annulus_x64_y64_ri20_ro50.tif"/><reconstruction absolute_filename="/home/empad/empad_projects/STO/21/scan_roi0_sum_annulus_x64_y64_ri20_ro50.tif"/><reconstruction absolute_filename="/home/empad/empad_projects/STO/21/scan_roi3_sum_annulus_x64_y64_ri20_ro50.tif"/><reconstruction absolute_filename="/home/empad/empad_projects/STO/21/scan_roi1_sum_annulus_x64_y64_ri20_ro50.tif"/><reconstruction absolute_filename="/home/empad/empad_projects/STO/21/scan_roi2_sum_annulus_x64_y64_ri20_ro50.tif"/><reconstruction absolute_filename="/home/empad/empad_projects/STO/21/scan_roi3_lr_annulus_x64_y64_ri20_ro50.tif"/><reconstruction absolute_filename="/home/empad/empad_projects/STO/21/scan_roi3_cw_annulus_x64_y64_ri20_ro50.tif"/></reconstructions><raw_file filename="scan_x128_y128.raw"/><background_image timestamp="1621297017.470743" absolute_filename=" /home/empad/backgrounds/img_exp_1_frm_0_cnt_1000_0002"/><exposure_time>1</exposure_time><post_exposure_time>0</post_exposure_time><camserver_parameters><camserver_acquisition_pending>False</camserver_acquisition_pending><camserver_inserted>True</camserver_inserted><camserver_scan_focus>False</camserver_scan_focus><camserver_poweron>True</camserver_poweron><camserver_video>False</camserver_video><camserver_scan_enabled>True</camserver_scan_enabled><camserver_poweron_done>True</camserver_poweron_done><camserver_connected>True</camserver_connected></camserver_parameters><scan_parameters mode="search"><scan_center_y>0.5</scan_center_y><scan_resolution_x>64</scan_resolution_x><scan_size>0.6</scan_size><scan_center_x>0.5</scan_center_x><scan_resolution_y>64</scan_resolution_y></scan_parameters><scan_parameters mode="acquire"><scan_center_y>0.5</scan_center_y><scan_resolution_x>128</scan_resolution_x><scan_size>1.0</scan_size><scan_center_x>0.5</scan_center_x><scan_resolution_y>128</scan_resolution_y></scan_parameters><reconstruction_parameters><active_roimaskidx>0</active_roimaskidx><roimask roi_idx="0"><roi_mode>3</roi_mode><center_x>64</center_x><center_y>64</center_y><radius_inner>20</radius_inner><radius_outer>50</radius_outer></roimask><roimask roi_idx="1"><roi_mode>3</roi_mode><center_x>64</center_x><center_y>64</center_y><radius_inner>20</radius_inner><radius_outer>50</radius_outer></roimask><roimask roi_idx="2"><roi_mode>3</roi_mode><center_x>64</center_x><center_y>64</center_y><radius_inner>20</radius_inner><radius_outer>50</radius_outer></roimask><roimask roi_idx="3"><roi_mode>3</roi_mode><center_x>64</center_x><center_y>64</center_y><radius_inner>20</radius_inner><radius_outer>50</radius_outer></roimask></reconstruction_parameters><temperature_controller><controller_enabled>True</controller_enabled><temperature_setpoint>-5.00</temperature_setpoint><temperature_value>-5.00</temperature_value><peltier_current>-2.290</peltier_current></temperature_controller><grid_measurements><controller_enabled>True</controller_enabled><bias_setpoint>150</bias_setpoint><bias_voltage>150.04</bias_voltage><bias_current>3.306</bias_current></grid_measurements><iom_measurements><scan_rotation>-0.09965830028887623</scan_rotation><is_column_in_stem_mode>True</is_column_in_stem_mode><spot_size_index>9</spot_size_index><full_scan_field_of_view><scale_factor>0.72</scale_factor><x>9.266641250196717e-09</x><y>9.266641250196717e-09</y></full_scan_field_of_view><nominal_camera_length>0.1159</nominal_camera_length><high_voltage_state>True</high_voltage_state><high_voltage>300000.0</high_voltage><vacuum_state>Ready</vacuum_state></iom_measurements></root>
diff --git a/hyperspy/tests/io/test_empad.py b/hyperspy/tests/io/test_empad.py
@@ -17,13 +17,14 @@
 # along with  HyperSpy.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import struct
 
 import numpy as np
 import pytest
 import traits.api as t
 
 import hyperspy.api as hs
+from hyperspy.io_plugins.empad import _parse_xml
+
 
 DATA_DIR = os.path.join(os.path.dirname(__file__), "empad_data")
 FILENAME_STACK_RAW = os.path.join(DATA_DIR, "series_x10.raw")
@@ -53,6 +54,7 @@ def teardown_module():
 
 @pytest.mark.parametrize("lazy", (False, True))
 def test_read_stack(lazy):
+    # xml file version 0.51 211118
     s = hs.load(os.path.join(DATA_DIR, "stack_images.xml"), lazy=lazy)
     assert s.data.dtype == "float32"
     ref_data = np.arange(166400).reshape((10, 130, 128))[..., :128, :]
@@ -77,6 +79,7 @@ def test_read_stack(lazy):
 
 @pytest.mark.parametrize("lazy", (False, True))
 def test_read_map(lazy):
+    # xml file version 0.51 211118
     s = hs.load(os.path.join(DATA_DIR, "map4x4.xml"), lazy=lazy)
     assert s.data.dtype == "float32"
     ref_data = np.arange(266240).reshape((4, 4, 130, 128))[..., :128, :]
@@ -99,3 +102,12 @@ def test_read_map(lazy):
     assert s.metadata.General.date == "2019-06-06"
     assert s.metadata.General.time == "13:30:00.164675"
     assert s.metadata.Signal.signal_type == "electron_diffraction"
+
+
+def test_parse_xml_1_2_0():
+    # xml file version 1.2.0 (2020-10-29)
+    filename = os.path.join(DATA_DIR, "map128x128_version1.2.0.xml")
+    om, info = _parse_xml(filename)
+    assert info['scan_x'] == 128
+    assert info['scan_y'] == 128
+    assert info['raw_filename'] == 'scan_x128_y128.raw'
diff --git a/hyperspy/tests/signals/test_map_method.py b/hyperspy/tests/signals/test_map_method.py
@@ -423,3 +423,21 @@ def test_map_ufunc(caplog):
     assert np.log(s) == s.map(np.log)
     np.testing.assert_allclose(s.data, np.log(data))
     assert "can direcly operate on hyperspy signals" in caplog.records[0].message
+
+
+class TestLazyNavChunkSize1:
+    @staticmethod
+    def afunction(input_data):
+        return np.array([1, 2, 3])
+
+    def test_signal2d(self):
+        dask_array = da.zeros((10, 15, 32, 32), chunks=(1, 1, 32, 32))
+        s = hs.signals.Signal2D(dask_array).as_lazy()
+        s_out = s.map(self.afunction, inplace=False, parallel=False, ragged=True)
+        s_out.compute()
+
+    def test_signal1d(self):
+        dask_array = da.zeros((10, 15, 32), chunks=(1, 1, 32))
+        s = hs.signals.Signal1D(dask_array).as_lazy()
+        s_out = s.map(self.afunction, inplace=False, parallel=False, ragged=True)
+        s_out.compute()
diff --git a/releasing_guide.md b/releasing_guide.md
@@ -21,7 +21,6 @@ Create a PR to the `RELEASE_next_patch` branch and go through the following step
 **Post-release action**
 - Increment the version and set it back to dev: `vx.y.z.dev0`
 - Update version in other branches when necessary
-- Prepare `CHANGES.rst` for development
 - Merge the PR
 
 Follow-up

diff --git a/setup.py b/setup.py
@@ -87,7 +87,7 @@
     "tests": ["pytest>=3.6", "pytest-mpl", "pytest-xdist", "pytest-rerunfailures", "pytest-instafail", "matplotlib>=3.1"],
     "coverage":["pytest-cov", "codecov"],
     # required to build the docs
-    "build-doc": ["sphinx>=1.7", "sphinx_rtd_theme", "sphinx-toggleprompt", "sphinxcontrib-mermaid"],
+    "build-doc": ["sphinx>=1.7", "sphinx_rtd_theme", "sphinx-toggleprompt", "sphinxcontrib-mermaid", "sphinxcontrib-towncrier"],
 }
 
 # Don't include "tests" and "docs" requirements since "all" is designed to be

diff --git a/upcoming_changes/2709.enhancements.rst b/upcoming_changes/2709.enhancements.rst
@@ -1,3 +1,3 @@
 Use ``importlib_metadata`` instead of ``pkg_resources`` for extensions
 registration to speed up the import process and making it possible to install
-extensions and use them without restarting the python session.
+extensions and use them without restarting the python session
diff --git a/upcoming_changes/2711.enhancements.rst b/upcoming_changes/2711.enhancements.rst
@@ -1 +1 @@
-Don't import hyperspy extensions when registering extensions.
+Don't import hyperspy extensions when registering extensions
diff --git a/upcoming_changes/2713.maintenance.rst b/upcoming_changes/2713.maintenance.rst
@@ -0,0 +1 @@
+Add drone CI to test arm64 platform
diff --git a/upcoming_changes/2714.maintenance.rst b/upcoming_changes/2714.maintenance.rst
@@ -0,0 +1 @@
+Fix latex doc build on github actions
diff --git a/upcoming_changes/2717.maintenance.rst b/upcoming_changes/2717.maintenance.rst
@@ -1 +1 @@
-Use towncrier to generate changelog automatically.
+Use towncrier to generate changelog automatically
diff --git a/upcoming_changes/2720.bugfix.rst b/upcoming_changes/2720.bugfix.rst
@@ -0,0 +1 @@
+Fix ROI snapping regression 
diff --git a/upcoming_changes/2722.maintenance.rst b/upcoming_changes/2722.maintenance.rst
@@ -0,0 +1 @@
+Fix test suite to support dask 2021.4.1
diff --git a/upcoming_changes/2724.enhancements.rst b/upcoming_changes/2724.enhancements.rst
@@ -0,0 +1 @@
+Improve docstrings of various fitting methods
diff --git a/upcoming_changes/2729.bugfix.rst b/upcoming_changes/2729.bugfix.rst
@@ -0,0 +1 @@
+Fix :py:meth:`~._signals.signal1d.Signal1D.shift1D` regression with navigation dimension larger than one
diff --git a/upcoming_changes/2734.bugfix.rst b/upcoming_changes/2734.bugfix.rst
@@ -0,0 +1 @@
+Fix disconnecting events when closing figure and :py:meth:`~._signals.signal1d.Signal1D.remove_background` is active
diff --git a/upcoming_changes/2748.bugfix.rst b/upcoming_changes/2748.bugfix.rst
@@ -0,0 +1 @@
+Fix :py:meth:`~.signal.BaseSignal.map` regression of lazy signal with navigation chunks of size of 1
diff --git a/upcoming_changes/2750.enhancements.rst b/upcoming_changes/2750.enhancements.rst
@@ -0,0 +1 @@
+Improve speed of :py:meth:`~._signals.signal1d.Signal1D.shift1D`
diff --git a/upcoming_changes/2757.enhancements.rst b/upcoming_changes/2757.enhancements.rst
@@ -0,0 +1 @@
+Add support for recent EMPAD file; scanning size wasn't parsed.
diff --git a/upcoming_changes/2758.maintenance.rst b/upcoming_changes/2758.maintenance.rst
@@ -0,0 +1 @@
+Generate changelog when building doc to keep the changelog of the development doc up to date on https://hyperspy.readthedocs.io/en/latest
diff --git a/upcoming_changes/2759.maintenance.rst b/upcoming_changes/2759.maintenance.rst
@@ -0,0 +1 @@
+Use mamba and conda-forge channel on azure pipeline
diff --git a/upcoming_changes/2760.bugfix.rst b/upcoming_changes/2760.bugfix.rst
@@ -0,0 +1 @@
+Fix unclear error message when reading a hspy file saved using blosc compression and `hdf5plugin` hasn't been imported previously