Release version 1.3.1

CHANGELOG.md

@@ -1,3 +1,13 @@
+## [Version 1.3.1] - 2022-11-23
+
+- Sentinel Hub IO tasks now support a custom timestamp filtration via `timestamp_filter` parameter.
+- `MergeFeatureTask` now supports the `axis` parameter.
+- Fix minor issues with the coregistration module.
+- Prepare for future removal of `sentinelhub.os_utils`.
+- Fix type annotations after `mypy` update.
+- Improvements to tests and various minor changes.
+
+
 ## [Version 1.3.0] - 2022-10-06
 
 - (**codebreaking**) Adapted Sentinel Hub tasks to `sentinelhub-py 3.8.0` which switched to Catalog 1.0.0.

core/eolearn/core/__init__.py

@@ -32,4 +32,4 @@
 from .utils.parallelize import execute_with_mp_lock, join_futures, join_futures_iter, parallelize
 from .utils.parsing import FeatureParser
 
-__version__ = "1.3.0"
+__version__ = "1.3.1"

core/eolearn/core/core_tasks.py

@@ -508,11 +508,11 @@ def zip_method(self, *f):
 
 
 class MergeFeatureTask(ZipFeatureTask):
-    """Merges multiple features together by concatenating their data along the last axis."""
+    """Merges multiple features together by concatenating their data along the specified axis."""
 
-    def zip_method(self, *f: np.ndarray, dtype: Union[None, np.dtype, type] = None) -> np.ndarray:
-        """Concatenates the data of features along the last axis."""
-        return np.concatenate(f, axis=-1, dtype=dtype)  # pylint: disable=unexpected-keyword-arg
+    def zip_method(self, *f: np.ndarray, dtype: Union[None, np.dtype, type] = None, axis: int = -1) -> np.ndarray:
+        """Concatenates the data of features along the specified axis."""
+        return np.concatenate(f, axis=axis, dtype=dtype)  # pylint: disable=unexpected-keyword-arg
 
 
 class ExtractBandsTask(MapFeatureTask):

core/eolearn/core/eodata.py

@@ -175,7 +175,7 @@ class _FeatureDictGeoDf(_FeatureDict[gpd.GeoDataFrame]):
     """_FeatureDict object specialized for GeoDataFrames."""
 
     def __init__(self, feature_dict: Dict[str, gpd.GeoDataFrame], feature_type: FeatureType):
-        if not feature_type.is_vector:
+        if not feature_type.is_vector():
             raise ValueError(f"Feature type {feature_type} does not represent a vector feature.")
         super().__init__(feature_dict, feature_type)
 

core/eolearn/core/eodata_io.py

@@ -16,6 +16,7 @@
 import datetime
 import gzip
 import json
+import platform
 import warnings
 from abc import ABCMeta, abstractmethod
 from collections import defaultdict
@@ -50,7 +51,6 @@
 
 from sentinelhub import CRS, BBox, Geometry, MimeType
 from sentinelhub.exceptions import SHUserWarning
-from sentinelhub.os_utils import sys_is_windows
 
 from .constants import FeatureType, FeatureTypeSet, OverwritePermission
 from .utils.parsing import FeatureParser, FeaturesSpecification
@@ -85,7 +85,7 @@ def save_eopatch(
         _check_letter_case_collisions(eopatch_features, fs_features)
         _check_add_only_permission(eopatch_features, fs_features)
 
-    elif sys_is_windows() and overwrite_permission is OverwritePermission.OVERWRITE_FEATURES:
+    elif platform.system() == "Windows" and overwrite_permission is OverwritePermission.OVERWRITE_FEATURES:
         _check_letter_case_collisions(eopatch_features, fs_features)
 
     else:

core/eolearn/tests/test_core_tasks.py

@@ -308,7 +308,8 @@ def test_move_feature():
     assert "MTless2" in patch_dst[FeatureType.MASK_TIMELESS]
 
 
-def test_merge_features():
+@pytest.mark.parametrize("axis", (0, -1))
+def test_merge_features(axis):
     patch = EOPatch()
 
     shape = (10, 5, 5, 3)
@@ -332,10 +333,10 @@ def test_merge_features():
     for feat, dat in zip(features, data):
         patch = AddFeatureTask(feat)(patch, dat)
 
-    patch = MergeFeatureTask(features[:3], (FeatureType.MASK, "merged"))(patch)
-    patch = MergeFeatureTask(features[3:], (FeatureType.MASK_TIMELESS, "merged_timeless"))(patch)
+    patch = MergeFeatureTask(features[:3], (FeatureType.MASK, "merged"), axis=axis)(patch)
+    patch = MergeFeatureTask(features[3:], (FeatureType.MASK_TIMELESS, "merged_timeless"), axis=axis)(patch)
 
-    expected = np.concatenate([patch[f] for f in features[:3]], axis=-1)
+    expected = np.concatenate([patch[f] for f in features[:3]], axis=axis)
 
     assert np.array_equal(patch.mask["merged"], expected)
 

core/eolearn/tests/test_utils/test_parsing.py

@@ -170,7 +170,7 @@ def test_allowed_feature_types(test_input: FeaturesSpecification, allowed_types:
         FeatureParser(features=test_input, allowed_feature_types=allowed_types)
 
 
-@pytest.fixture(name="eopatch", scope="session")
+@pytest.fixture(name="eopatch", scope="module")
 def eopatch_fixture():
     return EOPatch(
         data=dict(data=np.zeros((2, 2, 2, 2)), CLP=np.zeros((2, 2, 2, 2))),  # name duplication intentional

coregistration/eolearn/coregistration/__init__.py

@@ -4,4 +4,4 @@
 
 from .coregistration import ECCRegistrationTask, InterpolationType, PointBasedRegistrationTask, RegistrationTask
 
-__version__ = "1.3.0"
+__version__ = "1.3.1"

coregistration/eolearn/coregistration/coregistration.py

@@ -331,7 +331,7 @@ def register(self, src, trg, trg_mask=None, src_mask=None):
         # Initialise matrix and failed registrations flag
         warp_matrix = None
         # Initiate point detector
-        ptdt = cv2.xfeatures2d.SIFT_create() if self.params["Descriptor"] == "SIFT" else cv2.xfeatures2d.SURF_create()
+        ptdt = cv2.SIFT_create() if self.params["Descriptor"] == "SIFT" else cv2.SURF_create()
         # create BFMatcher object
         bf_matcher = cv2.BFMatcher(cv2.NORM_L1, crossCheck=True)
         # find the key points and descriptors with SIFT

features/eolearn/features/__init__.py

@@ -38,4 +38,4 @@
     AddSpatioTemporalFeaturesTask,
 )
 
-__version__ = "1.3.0"
+__version__ = "1.3.1"

features/eolearn/features/feature_manipulation.py

@@ -15,14 +15,15 @@
 import datetime as dt
 import logging
 from functools import partial
-from typing import Any, Callable, Dict, Iterable, List, Tuple, Union
+from typing import Any, Callable, Iterable, List, Tuple, Union
 
 import numpy as np
 from geopandas import GeoDataFrame
 
-from sentinelhub import BBox, bbox_to_dimensions
+from sentinelhub import bbox_to_dimensions
 
 from eolearn.core import EOPatch, EOTask, FeatureType, FeatureTypeSet, MapFeatureTask
+from eolearn.core.utils.parsing import FeaturesSpecification
 
 from .utils import ResizeLib, ResizeMethod, ResizeParam, spatially_resize_image
 
@@ -259,48 +260,40 @@ def map_method(self, feature: np.ndarray, slope: float, intercept: float) -> np.
 class SpatialResizeTask(EOTask):
     """Resizes the specified spatial features of EOPatch."""
 
-    def _process_resize_parameters(
-        self, resize_parameters: Tuple[ResizeParam, Tuple[float, float]], bbox: BBox
-    ) -> Dict[str, Tuple[float, float]]:
-        resize_type, resize_type_param = resize_parameters
-        resize_type = ResizeParam(resize_type)
-
-        if resize_type == ResizeParam.RESOLUTION:
-            new_size = bbox_to_dimensions(bbox, resize_type_param)
-            return {ResizeParam.NEW_SIZE.value: new_size}
-
-        return {resize_type.value: resize_type_param}
-
     def __init__(
         self,
         *,
         resize_parameters: Tuple[ResizeParam, Tuple[float, float]],
-        features: Any = ...,
+        features: FeaturesSpecification = ...,
         resize_method: ResizeMethod = ResizeMethod.LINEAR,
         resize_library: ResizeLib = ResizeLib.PIL,
     ):
         """
-        :param features: The specification of feature for which to perform resizing. Must be supported by the
-            :class:`FeatureParser<eolearn.core.utilities.FeatureParser>`. Features can be renamed, see `FeatureParser`
-            documentation.
-        :param new_size: New size of the data (height, width)
-        :param scale_factors: Factors (f_height, f_width) by which to resize the image
+        :param features: Which features to resize. Supports new names for features.
+        :param resize_parameters: Instructions on how to perform the resizing process. For example use:
+            `['new_size', [500, 1000]]` to resize the data to size (500, 1000),
+            `['resolution', [10, 20]]` for changing resolution from 10 to 20,
+            `['scale_factors', [3, 3]]` to make the data three times larger.
         :param resize_method: Interpolation method used for resizing.
         :param resize_library: Which Python library to use for resizing. Default is PIL, as it supports all dtypes and
             features anti-aliasing. For cases where execution speed is crucial one can use CV2.
         """
         self.features = features
-        self.resize_parameters = resize_parameters
+        self.parameter_kind = ResizeParam(resize_parameters[0])
+        self.parameter_values = resize_parameters[1]
 
         self.resize_function = partial(
-            spatially_resize_image,
-            resize_method=resize_method,
-            resize_library=resize_library,
+            spatially_resize_image, resize_method=resize_method, resize_library=resize_library
         )
 
     def execute(self, eopatch: EOPatch) -> EOPatch:
-        resize_params = self._process_resize_parameters(self.resize_parameters, eopatch.bbox)
+        if self.parameter_kind == ResizeParam.RESOLUTION:
+            new_size = bbox_to_dimensions(eopatch.bbox, self.parameter_values)
+            resize_fun_kwargs = {ResizeParam.NEW_SIZE.value: new_size}
+        else:
+            resize_fun_kwargs = {self.parameter_kind.value: self.parameter_values}
+
         for ftype, fname, new_name in self.parse_renamed_features(self.features, eopatch=eopatch):
             if ftype.is_spatial() and ftype.is_raster():
-                eopatch[ftype, new_name] = self.resize_function(eopatch[ftype, fname], **resize_params)
+                eopatch[ftype, new_name] = self.resize_function(eopatch[ftype, fname], **resize_fun_kwargs)
         return eopatch

features/eolearn/tests/conftest.py

@@ -8,6 +8,8 @@
 
 from eolearn.core import EOPatch
 
+pytest.register_assert_rewrite("sentinelhub.testing_utils")  # makes asserts in helper functions work with pytest
+
 EXAMPLE_DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", "example_data")
 EXAMPLE_EOPATCH_PATH = os.path.join(EXAMPLE_DATA_PATH, "TestEOPatch")
 

features/eolearn/tests/test_blob.py

@@ -11,51 +11,50 @@
 import copy
 import sys
 
-import numpy as np
 import pytest
-from numpy.testing import assert_array_equal
 from pytest import approx
 from skimage.feature import blob_dog
 
+from sentinelhub.testing_utils import test_numpy_data
+
 from eolearn.core import FeatureType
-from eolearn.core.eodata_io import FeatureIO
 from eolearn.features import BlobTask, DoGBlobTask, DoHBlobTask, LoGBlobTask
 
 FEATURE = (FeatureType.DATA, "NDVI", "blob")
+BLOB_FEATURE = (FeatureType.DATA, "blob")
 
 
-def test_blob_feature(small_ndvi_eopatch):
+def test_dog_blob_task(small_ndvi_eopatch):
     eopatch = small_ndvi_eopatch
     BlobTask(FEATURE, blob_dog, sigma_ratio=1.6, min_sigma=1, max_sigma=30, overlap=0.5, threshold=0)(eopatch)
     DoGBlobTask((FeatureType.DATA, "NDVI", "blob_dog"), threshold=0)(eopatch)
-    assert eopatch.data["blob"] == approx(
-        eopatch.data["blob_dog"]
-    ), "DoG derived class result not equal to base class result"
+    assert eopatch[BLOB_FEATURE] == approx(eopatch.data["blob_dog"])
 
 
 BLOB_TESTS = [
-    [DoGBlobTask(FEATURE, threshold=0), 0.0, 37.9625, 0.0854, 0.0],
+    (DoGBlobTask(FEATURE, threshold=0), {"exp_min": 0.0, "exp_max": 37.9625, "exp_mean": 0.08545, "exp_median": 0.0}),
 ]
 if sys.version_info >= (3, 8):  # For Python 3.7 scikit-image returns less accurate result for this test
-    BLOB_TESTS.append([DoHBlobTask(FEATURE, num_sigma=5, threshold=0), 0.0, 21.9203, 0.05807, 0.0])
-    BLOB_TESTS.append([LoGBlobTask(FEATURE, log_scale=True, threshold=0), 0, 42.4264, 0.0977, 0.0])
-
-
-@pytest.mark.parametrize("task, expected_min, expected_max, expected_mean, expected_median", BLOB_TESTS)
-def test_blob(small_ndvi_eopatch, task, expected_min, expected_max, expected_mean, expected_median):
+    BLOB_TESTS.extend(
+        [
+            (
+                DoHBlobTask(FEATURE, num_sigma=5, threshold=0),
+                {"exp_min": 0.0, "exp_max": 21.9203, "exp_mean": 0.05807, "exp_median": 0.0},
+            ),
+            (
+                LoGBlobTask(FEATURE, log_scale=True, threshold=0),
+                {"exp_min": 0, "exp_max": 42.4264, "exp_mean": 0.09767, "exp_median": 0.0},
+            ),
+        ]
+    )
+
+
+@pytest.mark.parametrize("task, expected_statistics", BLOB_TESTS)
+def test_blob_task(small_ndvi_eopatch, task, expected_statistics):
     eopatch = copy.deepcopy(small_ndvi_eopatch)
     task.execute(eopatch)
 
-    # Test that no other features were modified
-    for feature, value in small_ndvi_eopatch.data.items():
-        if isinstance(value, FeatureIO):
-            value = value.load()
-        assert_array_equal(value, eopatch.data[feature], err_msg=f"EOPatch data feature '{feature}' has changed")
-
-    delta = 1e-4
+    test_numpy_data(eopatch[BLOB_FEATURE], exp_shape=(10, 20, 20, 1), **expected_statistics, delta=1e-4)
 
-    blob = eopatch.data[FEATURE[-1]]
-    assert np.min(blob) == approx(expected_min, abs=delta)
-    assert np.max(blob) == approx(expected_max, abs=delta)
-    assert np.mean(blob) == approx(expected_mean, abs=delta)
-    assert np.median(blob) == approx(expected_median, abs=delta)
+    del eopatch[BLOB_FEATURE]
+    assert small_ndvi_eopatch == eopatch, "Other features of the EOPatch were affected."

features/eolearn/tests/test_haralick.py

@@ -10,56 +10,38 @@
 
 import numpy as np
 import pytest
-from numpy.testing import assert_array_equal
-from pytest import approx
+
+from sentinelhub.testing_utils import test_numpy_data
 
 from eolearn.core import FeatureType
-from eolearn.core.eodata_io import FeatureIO
 from eolearn.features import HaralickTask
 
 FEATURE = (FeatureType.DATA, "NDVI", "haralick")
+OUTPUT_FEATURE = (FeatureType.DATA, "haralick")
 
 
 @pytest.mark.parametrize(
-    "task, expected_min, expected_max, expected_mean, expected_median",
+    "task, expected_statistics",
     (
         [
             HaralickTask(FEATURE, texture_feature="contrast", angle=0, levels=255, window_size=3),
-            3.5,
-            9079.0,
-            965.8295,
-            628.5833,
+            {"exp_min": 3.5, "exp_max": 9079.0, "exp_mean": 965.8295, "exp_median": 628.5833},
         ],
         [
             HaralickTask(FEATURE, texture_feature="sum_of_square_variance", angle=np.pi / 2, levels=8, window_size=5),
-            0.96899,
-            48.7815,
-            23.0229,
-            23.8987,
+            {"exp_min": 0.96899, "exp_max": 48.7815, "exp_mean": 23.0229, "exp_median": 23.8987},
         ],
         [
             HaralickTask(FEATURE, texture_feature="sum_entropy", angle=-np.pi / 2, levels=8, window_size=7),
-            0,
-            1.7463,
-            0.5657,
-            0.5055,
+            {"exp_min": 0, "exp_max": 1.7463, "exp_mean": 0.5657, "exp_median": 0.50558},
         ],
     ),
 )
-def test_haralick(small_ndvi_eopatch, task, expected_min, expected_max, expected_mean, expected_median):
+def test_haralick(small_ndvi_eopatch, task, expected_statistics):
     eopatch = copy.deepcopy(small_ndvi_eopatch)
     task.execute(eopatch)
 
-    # Test that no other features were modified
-    for feature, value in small_ndvi_eopatch.data.items():
-        if isinstance(value, FeatureIO):
-            value = value.load()
-        assert_array_equal(value, eopatch.data[feature], err_msg=f"EOPatch data feature '{feature}' has changed")
-
-    delta = 1e-4
+    test_numpy_data(eopatch[OUTPUT_FEATURE], exp_shape=(10, 20, 20, 1), **expected_statistics, delta=1e-4)
 
-    haralick = eopatch.data["haralick"]
-    assert np.min(haralick) == approx(expected_min, abs=delta)
-    assert np.max(haralick) == approx(expected_max, abs=delta)
-    assert np.mean(haralick) == approx(expected_mean, abs=delta)
-    assert np.median(haralick) == approx(expected_median, abs=delta)
+    del eopatch[OUTPUT_FEATURE]
+    assert small_ndvi_eopatch == eopatch, "Other features of the EOPatch were affected."