Merge pull request #2866 from ericpre/fix_CI

CI fixes and improvements

.github/workflows/tests.yml

@@ -23,7 +23,7 @@ jobs:
           - os: ubuntu
             PYTHON_VERSION: 3.7
             OLDEST_SUPPORTED_VERSION: true
-            DEPENDENCIES: matplotlib==3.1.0 numpy==1.17.1 scipy==1.1 imagecodecs==2020.1.31 tifffile==2020.2.16 dask==2.1.0 scikit-image==0.15
+            DEPENDENCIES: matplotlib==3.1.3 numpy==1.17.1 scipy==1.1 imagecodecs==2020.1.31 tifffile==2020.2.16 dask==2.11.0 distributed==2.11.0 scikit-image==0.15
             PIP_SELECTOR: '[all, tests, coverage]'
             PYTEST_ARGS_COVERAGE: --cov=. --cov-report=xml
             LABEL: -oldest
@@ -68,7 +68,11 @@ jobs:
         if: ${{ matrix.OLDEST_SUPPORTED_VERSION }}
         run: |
           pip install ${{ matrix.DEPENDENCIES }}
-          
+
+      - name: Pip list
+        run: |
+          pip list
+
       - name: Run test suite
         run: |
           pytest ${{ env.PYTEST_ARGS }} ${{ matrix.PYTEST_ARGS_COVERAGE }}
@@ -89,8 +93,8 @@ jobs:
 
       - uses: ericpre/sphinx-action@latest_sphinx
         with:
-
-          pre-build-command: "apt-get update -y && apt-get install -y ${{ env.BUILD_DEPS }} ${{ env.LATEX_DEPS }}  && pip install -e .[all,build-doc]"
+          # once numba 0.55 is release, remove "--pre" option to "pip install"
+          pre-build-command: "apt-get update -y && apt-get install -y ${{ env.BUILD_DEPS }} ${{ env.LATEX_DEPS }} && pip install --pre .[all,build-doc]"
           build-command: make html
           docs-folder: doc/
 

azure-pipelines.yml

@@ -56,6 +56,8 @@ pool:
   vmImage: '$(vmImage)'
 
 steps:
+- checkout: self
+  fetchDepth: 1 # Fetch only one commit
 - template: azure_pipelines/clone_ci-scripts_repo.yml@templates
 - template: azure_pipelines/install_mambaforge.yml@templates
 - template: azure_pipelines/activate_conda.yml@templates

conda_environment.yml

@@ -2,7 +2,7 @@ name: test_env
 channels:
 - conda-forge
 dependencies:
-- dask-core >2.1
+- dask-core >=2.11
 - dill
 # dask-core < 2021.3.1 doesn't have fsspec as dependency
 - fsspec

conda_environment_dev.yml

@@ -4,7 +4,7 @@ channels:
 dependencies:
 # We pin freetype and matplotlib for the image comparison
 - freetype=2.9
-- matplotlib-base=3.1
+- matplotlib-base=3.1.3
 - cython
 - pytest
 - pytest-mpl

hyperspy/misc/array_tools.py

@@ -182,17 +182,9 @@ def rebin(a, new_shape=None, scale=None, crop=True, dtype=None):
                 2 * i + 1 for i in range(lenShape)), dtype=dtype)
         else:
             try:
-                kwargs = {}
-                if LooseVersion(dask.__version__) >= LooseVersion('2.11.0'):
-                    kwargs['dtype'] = dtype
-                elif dtype is not None:
-                    raise ValueError(
-                        'Using the dtype argument for lazy signal requires '
-                        'dask >= 2.11.0.'
-                        )
                 return da.coarsen(np.sum, a,
                                   {i: int(f) for i, f in enumerate(scale)},
-                                  **kwargs)
+                                  dtype=dtype)
             # we provide slightly better error message in hyperspy context
             except ValueError:
                 raise ValueError(

hyperspy/tests/io/test_jeol.py

@@ -16,13 +16,23 @@
 # You should have received a copy of the GNU General Public License
 # along with  HyperSpy.  If not, see <http://www.gnu.org/licenses/>.
 
+import gc
 from pathlib import Path
+
 import pytest
 import numpy as np
 
 import hyperspy.api as hs
 
 
+def teardown_module(module):
+    """
+    Run a garbage collection cycle at the end of the test of this module
+    to avoid any memory issue when continuing running the test suite.
+    """
+    gc.collect()
+
+
 TESTS_FILE_PATH = Path(__file__).resolve().parent / 'JEOL_files'
 TESTS_FILE_PATH2 = TESTS_FILE_PATH / 'InvalidFrame'
 
@@ -150,7 +160,7 @@ def test_load_datacube(SI_dtype):
 
 def test_load_datacube_rebin_energy():
     filename = TESTS_FILE_PATH / 'Sample' / '00_View000' / test_files[7]
-    s = hs.load(filename)
+    s = hs.load(filename, cutoff_at_kV=1)
     s_sum = s.sum()
 
     ref_data = hs.signals.Signal1D(
@@ -171,6 +181,7 @@ def test_load_datacube_rebin_energy():
 
 
 def test_load_datacube_cutoff_at_kV():
+    gc.collect()
     cutoff_at_kV = 10.
     filename = TESTS_FILE_PATH / 'Sample' / '00_View000' / test_files[7]
     s = hs.load(filename, cutoff_at_kV=None)
@@ -359,9 +370,7 @@ def test_pts_lazy():
 
 
 def test_pts_frame_shift():
-    dir = TESTS_FILE_PATH2
-    dir2p = dir / 'Sample' / '00_Dummy-Data'
-    file = str(dir2p / test_files2[16])
+    file = TESTS_FILE_PATH2 / 'Sample' / '00_Dummy-Data' / test_files2[16]
 
     # without frame shift
     ref = hs.load(file, read_em_image=True, only_valid_data=False,

hyperspy/tests/learn/test_bss.py

@@ -18,7 +18,6 @@
 
 import numpy as np
 import pytest
-from distutils.version import LooseVersion
 
 from hyperspy._signals.signal1d import Signal1D
 from hyperspy._signals.signal2d import Signal2D
@@ -352,8 +351,6 @@ def setup_method(self, method):
         self.mask_nav = mask_nav
         self.mask_sig = mask_sig
 
-    @pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion("1.17"),
-                        reason="derivative of lazy signal isn't supported with numpy < 1.17.0")
     def test_diff_axes_string_with_mask(self):
         self.s.learning_results.factors[5, :] = np.nan
         factors = self.s.get_decomposition_factors().inav[:3]
@@ -391,8 +388,6 @@ def test_diff_axes_string_with_mask(self):
             matrix, self.s.learning_results.unmixing_matrix, atol=1e-5
         )
 
-    @pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion("1.17"),
-                        reason="derivative of lazy signal isn't supported with numpy < 1.17.0")
     def test_diff_axes_string_without_mask(self):
         factors = self.s.get_decomposition_factors().inav[:3].derivative(
             axis="x", order=1)
@@ -407,8 +402,6 @@ def test_diff_axes_string_without_mask(self):
             matrix, self.s.learning_results.unmixing_matrix, atol=1e-3
         )
 
-    @pytest.mark.skipif(LooseVersion(np.__version__) < LooseVersion("1.17"),
-                        reason="derivative of lazy signal isn't supported with numpy < 1.17.0")
     def test_diff_axes_without_mask(self):
         factors = self.s.get_decomposition_factors().inav[:3].derivative(
             axis="y", order=1)

hyperspy/tests/misc/test_hist_tools.py

@@ -33,16 +33,13 @@ def generate_bad_toy_data():
     which previously caused a MemoryError when
     using the Freedman-Diaconis rule.
     """
-    ax1 = np.exp(-np.abs(np.arange(-30, 100, 0.02)))
-    ax2 = np.exp(-np.abs(np.arange(-40, 90, 0.02)))
+    ax1 = np.exp(-np.abs(np.arange(-30, 100, 0.05)))
     s1 = hs.signals.EELSSpectrum(ax1)
-    s2 = hs.signals.EELSSpectrum(ax2)
-    s1 = hs.stack([s1] * 5)
-    s2 = hs.stack([s2] * 5)
-    s1.align_zero_loss_peak(also_align=[s2])
+    s1 = hs.stack([s1] * 2)
     return s1
 
-@pytest.mark.parametrize("bins",[10,np.linspace(1,20,num=11)])
+
+@pytest.mark.parametrize("bins", [10, np.linspace(1, 20, num=11)])
 def test_types_of_bins(bins):
     s1 = generate_bad_toy_data()
     out = s1.get_histogram(bins)
@@ -95,7 +92,7 @@ def test_int_bins_logger_warning(self, caplog):
         assert out.data.shape == (250,)
         assert "Capping the number of bins" in caplog.text
 
-    @pytest.mark.parametrize("bins, size", [("scott", (106,)), (10, (10,))])
+    @pytest.mark.parametrize("bins, size", [("scott", (58,)), (10, (10,))])
     def test_working_bins(self, bins, size):
         out = self.s1.get_histogram(bins=bins)
         assert out.data.shape == size

hyperspy/tests/signals/test_3D.py

@@ -334,13 +334,6 @@ def test_rebin_invalid_dtype_args(self):
     def test_rebin_dtype(self, dtype):
         s = self.s
         s.change_dtype(np.uint8)
-        if s._lazy and LooseVersion(dask.__version__) < LooseVersion("2.11.0"):
-            if dtype is not None:
-                with pytest.raises(ValueError):
-                    _ = s.rebin(scale=(5, 2, 1), dtype=dtype)
-            # exit to skip the rest of the test
-            return
-
         s2 = s.rebin(scale=(5, 2, 1), dtype=dtype)
         if dtype == None:
             # np.sum default uses platform (un)signed interger (input dependent)

setup.py

@@ -46,7 +46,7 @@
 
 
 install_req = ['scipy>=1.1',
-               'matplotlib>=3.1.0,<3.5',
+               'matplotlib>=3.1.3,<3.5',
                'numpy>=1.17.1',
                'traits>=4.5.0',
                'natsort',
@@ -58,7 +58,7 @@
                'packaging',
                'python-dateutil>=2.5.0',
                'ipyparallel',
-               'dask[array]>=2.1.0',
+               'dask[array]>=2.11.0',
                # fsspec is missing from dask dependencies for dask < 2021.3.1
                'fsspec',
                'scikit-image>=0.15',
@@ -353,7 +353,10 @@ def __exit__(self, type, value, traceback):
                 'tests/io/hdf5_files/*.hspy',
                 'tests/io/JEOL_files/*',
                 'tests/io/JEOL_files/Sample/00_View000/*',
+                'tests/io/JEOL_files/InvalidFrame/*',
+                'tests/io/JEOL_files/InvalidFrame/Sample/00_Dummy-Data/*',
                 'tests/io/tiff_files/*.tif',
+                'tests/io/tiff_files/*.tif.gz',
                 'tests/io/tiff_files/*.dm3',
                 'tests/io/npy_files/*.npy',
                 'tests/io/unf_files/*.unf',

upcoming_changes/2866.maintenance.rst

@@ -0,0 +1 @@
+Continuous integration fixes and improvements; Bump minimal version requirement of dask to 2.11.0 and matplotlib to 3.1.3