From a42a89e764eb7c8b678c863a551ec38b49ab5598 Mon Sep 17 00:00:00 2001 From: jgersak Date: Tue, 11 Oct 2022 14:27:42 +0200 Subject: [PATCH 01/14] add scope fixture in tests --- core/eolearn/tests/test_utils/test_parsing.py | 2 +- visualization/eolearn/tests/test_eopatch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/eolearn/tests/test_utils/test_parsing.py b/core/eolearn/tests/test_utils/test_parsing.py index 664c23a16..cb8f20445 100644 --- a/core/eolearn/tests/test_utils/test_parsing.py +++ b/core/eolearn/tests/test_utils/test_parsing.py @@ -170,7 +170,7 @@ def test_allowed_feature_types(test_input: FeaturesSpecification, allowed_types: FeatureParser(features=test_input, allowed_feature_types=allowed_types) -@pytest.fixture(name="eopatch", scope="session") +@pytest.fixture(name="eopatch", scope="module") def eopatch_fixture(): return EOPatch( data=dict(data=np.zeros((2, 2, 2, 2)), CLP=np.zeros((2, 2, 2, 2))), # name duplication intentional diff --git a/visualization/eolearn/tests/test_eopatch.py b/visualization/eolearn/tests/test_eopatch.py index 1d2dd555c..6e7078044 100644 --- a/visualization/eolearn/tests/test_eopatch.py +++ b/visualization/eolearn/tests/test_eopatch.py @@ -17,7 +17,7 @@ from eolearn.visualization import PlotConfig -@pytest.fixture(name="eopatch") +@pytest.fixture(name="eopatch", scope="module") def eopatch_fixture(): path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", "example_data", "TestEOPatch") return EOPatch.load(path) From b1072d25de17d98f04eec2c4d44f3723c265d386 Mon Sep 17 00:00:00 2001 From: jgersak Date: Tue, 18 Oct 2022 13:51:55 +0200 Subject: [PATCH 02/14] Fix coregistration register opencv sift were moved to their main repo --- coregistration/eolearn/coregistration/coregistration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coregistration/eolearn/coregistration/coregistration.py b/coregistration/eolearn/coregistration/coregistration.py index ec1f5e912..1a23d1d51 100644 --- a/coregistration/eolearn/coregistration/coregistration.py +++ b/coregistration/eolearn/coregistration/coregistration.py @@ -331,7 +331,7 @@ def register(self, src, trg, trg_mask=None, src_mask=None): # Initialise matrix and failed registrations flag warp_matrix = None # Initiate point detector - ptdt = cv2.xfeatures2d.SIFT_create() if self.params["Descriptor"] == "SIFT" else cv2.xfeatures2d.SURF_create() + ptdt = cv2.SIFT_create() if self.params["Descriptor"] == "SIFT" else cv2.SURF_create() # create BFMatcher object bf_matcher = cv2.BFMatcher(cv2.NORM_L1, crossCheck=True) # find the key points and descriptors with SIFT From d80ed0f337bd369a5d607d42681de424467187fe Mon Sep 17 00:00:00 2001 From: jgersak Date: Wed, 2 Nov 2022 13:49:12 +0100 Subject: [PATCH 03/14] fix upcoming deprecation of SH-PY os_utils --- core/eolearn/core/eodata_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/eolearn/core/eodata_io.py b/core/eolearn/core/eodata_io.py index af1c2a5ff..b075dde41 100644 --- a/core/eolearn/core/eodata_io.py +++ b/core/eolearn/core/eodata_io.py @@ -16,6 +16,7 @@ import datetime import gzip import json +import platform import warnings from abc import ABCMeta, abstractmethod from collections import defaultdict @@ -50,7 +51,6 @@ from sentinelhub import CRS, BBox, Geometry, MimeType from sentinelhub.exceptions import SHUserWarning -from sentinelhub.os_utils import sys_is_windows from .constants import FeatureType, FeatureTypeSet, OverwritePermission from .utils.parsing import FeatureParser, FeaturesSpecification @@ -85,7 +85,7 @@ def save_eopatch( _check_letter_case_collisions(eopatch_features, fs_features) _check_add_only_permission(eopatch_features, fs_features) - elif sys_is_windows() and overwrite_permission is OverwritePermission.OVERWRITE_FEATURES: + elif platform.system() == "Windows" and overwrite_permission is OverwritePermission.OVERWRITE_FEATURES: _check_letter_case_collisions(eopatch_features, fs_features) else: From 958520bec544a9f48bbf6a69e5a3b39fdba3974e Mon Sep 17 00:00:00 2001 From: Ziga Luksic Date: Tue, 8 Nov 2022 09:33:16 +0100 Subject: [PATCH 04/14] fixes after mypy update --- core/eolearn/core/eodata.py | 2 +- requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/eolearn/core/eodata.py b/core/eolearn/core/eodata.py index d3c3868db..6d46b11da 100644 --- a/core/eolearn/core/eodata.py +++ b/core/eolearn/core/eodata.py @@ -175,7 +175,7 @@ class _FeatureDictGeoDf(_FeatureDict[gpd.GeoDataFrame]): """_FeatureDict object specialized for GeoDataFrames.""" def __init__(self, feature_dict: Dict[str, gpd.GeoDataFrame], feature_type: FeatureType): - if not feature_type.is_vector: + if not feature_type.is_vector(): raise ValueError(f"Feature type {feature_type} does not represent a vector feature.") super().__init__(feature_dict, feature_type) diff --git a/requirements-dev.txt b/requirements-dev.txt index f88e4b170..2ace71579 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ codecov hypothesis moto -mypy +mypy>=0.990 nbval pylint>=2.14.0 pytest>=7.0.0 From 248cc3fbbcddb918381dc1834af047b3a8ed3200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Wed, 9 Nov 2022 14:35:34 +0100 Subject: [PATCH 05/14] add axis parameter to MergeFeatureTask (#496) --- core/eolearn/core/core_tasks.py | 8 ++++---- core/eolearn/tests/test_core_tasks.py | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/core/eolearn/core/core_tasks.py b/core/eolearn/core/core_tasks.py index 6df8bb974..8b56f802d 100644 --- a/core/eolearn/core/core_tasks.py +++ b/core/eolearn/core/core_tasks.py @@ -508,11 +508,11 @@ def zip_method(self, *f): class MergeFeatureTask(ZipFeatureTask): - """Merges multiple features together by concatenating their data along the last axis.""" + """Merges multiple features together by concatenating their data along the specified axis.""" - def zip_method(self, *f: np.ndarray, dtype: Union[None, np.dtype, type] = None) -> np.ndarray: - """Concatenates the data of features along the last axis.""" - return np.concatenate(f, axis=-1, dtype=dtype) # pylint: disable=unexpected-keyword-arg + def zip_method(self, *f: np.ndarray, dtype: Union[None, np.dtype, type] = None, axis: int = -1) -> np.ndarray: + """Concatenates the data of features along the specified axis.""" + return np.concatenate(f, axis=axis, dtype=dtype) # pylint: disable=unexpected-keyword-arg class ExtractBandsTask(MapFeatureTask): diff --git a/core/eolearn/tests/test_core_tasks.py b/core/eolearn/tests/test_core_tasks.py index f8a9dec59..9536180e1 100644 --- a/core/eolearn/tests/test_core_tasks.py +++ b/core/eolearn/tests/test_core_tasks.py @@ -308,7 +308,8 @@ def test_move_feature(): assert "MTless2" in patch_dst[FeatureType.MASK_TIMELESS] -def test_merge_features(): +@pytest.mark.parametrize("axis", (0, -1)) +def test_merge_features(axis): patch = EOPatch() shape = (10, 5, 5, 3) @@ -332,10 +333,10 @@ def test_merge_features(): for feat, dat in zip(features, data): patch = AddFeatureTask(feat)(patch, dat) - patch = MergeFeatureTask(features[:3], (FeatureType.MASK, "merged"))(patch) - patch = MergeFeatureTask(features[3:], (FeatureType.MASK_TIMELESS, "merged_timeless"))(patch) + patch = MergeFeatureTask(features[:3], (FeatureType.MASK, "merged"), axis=axis)(patch) + patch = MergeFeatureTask(features[3:], (FeatureType.MASK_TIMELESS, "merged_timeless"), axis=axis)(patch) - expected = np.concatenate([patch[f] for f in features[:3]], axis=-1) + expected = np.concatenate([patch[f] for f in features[:3]], axis=axis) assert np.array_equal(patch.mask["merged"], expected) From 05990cbaec17b9b6b2e82f2bea6da3533a904a6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Mon, 14 Nov 2022 12:06:18 +0100 Subject: [PATCH 06/14] Improve type signatures in sh-io (#499) * improve type signatures in sh-io * remove redundant docstring type * revert to RawTimeIntervalType to have mypy help prevent codebreaks --- io/eolearn/io/sentinelhub_process.py | 65 +++++++++++++++++++++------- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/io/eolearn/io/sentinelhub_process.py b/io/eolearn/io/sentinelhub_process.py index 00f3921ff..853604eee 100644 --- a/io/eolearn/io/sentinelhub_process.py +++ b/io/eolearn/io/sentinelhub_process.py @@ -12,7 +12,7 @@ import datetime as dt import logging -from typing import Callable, List, Optional, Tuple, Union +from typing import Any, Callable, List, Optional, Tuple, Union import numpy as np @@ -35,9 +35,11 @@ parse_time_interval, serialize_time, ) -from sentinelhub.time_utils import RawTimeIntervalType +from sentinelhub.type_utils import RawTimeIntervalType from eolearn.core import EOPatch, EOTask, FeatureType, FeatureTypeSet +from eolearn.core.utils.parsing import FeatureSpec, FeaturesSpecification +from eolearn.core.utils.types import Literal LOGGER = logging.getLogger(__name__) @@ -87,9 +89,9 @@ def execute( self, eopatch: Optional[EOPatch] = None, bbox: Optional[BBox] = None, - time_interval: Optional[RawTimeIntervalType] = None, + time_interval: Optional[RawTimeIntervalType] = None, # should be kept at this to prevent code-breaks geometry: Optional[Geometry] = None, - ): + ) -> EOPatch: """Main execute method for the Process API tasks. The `geometry` is used only in conjunction with the `bbox` and does not act as a replacement.""" @@ -182,11 +184,19 @@ def _extract_data(self, eopatch, images, shape): """Extract data from the received images and assign them to eopatch features""" raise NotImplementedError("The _extract_data method should be implemented by the subclass.") - def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval, geometry): + def _build_requests( + self, + bbox: BBox, + size_x: int, + size_y: int, + timestamp: Optional[List[dt.datetime]], + time_interval: Optional[Tuple[dt.datetime, dt.datetime]], + geometry: Geometry, + ) -> List[SentinelHubRequest]: """Build requests""" raise NotImplementedError("The _build_requests method should be implemented by the subclass.") - def _get_timestamp(self, time_interval, bbox): + def _get_timestamp(self, time_interval: Optional[Tuple[dt.datetime, dt.datetime]], bbox: BBox) -> List[dt.datetime]: """Get the timestamp array needed as a parameter for downloading the images""" raise NotImplementedError("The _get_timestamp method should be implemented by the subclass.") @@ -197,7 +207,7 @@ class SentinelHubEvalscriptTask(SentinelHubInputBaseTask): # pylint: disable=too-many-arguments def __init__( self, - features: Optional[FeatureType] = None, + features: Optional[FeaturesSpecification] = None, evalscript: Optional[str] = None, data_collection: Optional[DataCollection] = None, size: Optional[Tuple[int, int]] = None, @@ -301,7 +311,15 @@ def _get_timestamp(self, time_interval, bbox): config=self.config, ) - def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval, geometry): + def _build_requests( + self, + bbox: BBox, + size_x: int, + size_y: int, + timestamp: List[dt.datetime], + time_interval: Optional[Tuple[dt.datetime, dt.datetime]], + geometry: Geometry, + ): """Defines request timestamps and builds requests. In case `timestamp` is either `None` or an empty list it still has to create at least one request in order to obtain back number of bands of responses.""" if timestamp: @@ -388,7 +406,7 @@ def __init__( cache_folder: Optional[str] = None, config: Optional[SHConfig] = None, max_threads: Optional[int] = None, - bands_dtype: Optional[List[str]] = None, + bands_dtype: Union[np.dtype, type] = None, single_scene: bool = False, mosaicking_order: Optional[Union[str, MosaickingOrder]] = None, upsampling: Optional[ResamplingType] = None, @@ -475,7 +493,7 @@ def _parse_requested_bands(self, bands, available_bands): ) return requested_bands - def generate_evalscript(self): + def generate_evalscript(self) -> str: """Generate the evalscript to be passed with the request, based on chosen bands""" evalscript = """ //VERSION=3 @@ -535,7 +553,15 @@ def _get_timestamp(self, time_interval, bbox): config=self.config, ) - def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval, geometry): + def _build_requests( + self, + bbox: BBox, + size_x: int, + size_y: int, + timestamp: Optional[List[dt.datetime]], + time_interval: Optional[Tuple[dt.datetime, dt.datetime]], + geometry: Geometry, + ) -> List[SentinelHubRequest]: """Build requests""" if timestamp is None: dates = [None] @@ -618,7 +644,12 @@ class SentinelHubDemTask(SentinelHubEvalscriptTask): DATA_TIMELESS EOPatch feature. """ - def __init__(self, feature=None, data_collection=DataCollection.DEM, **kwargs): + def __init__( + self, + feature: Union[None, str, FeatureSpec] = None, + data_collection: DataCollection = DataCollection.DEM, + **kwargs: Any, + ): if feature is None: feature = (FeatureType.DATA_TIMELESS, "dem") elif isinstance(feature, str): @@ -675,11 +706,15 @@ class SentinelHubSen2corTask(SentinelHubInputTask): * 11 - SNOW """ - def __init__(self, sen2cor_classification, data_collection=DataCollection.SENTINEL2_L2A, **kwargs): + def __init__( + self, + sen2cor_classification: Union[Literal["SCL", "CLD", "SNW"], List[Literal["SCL", "CLD", "SNW"]]], + data_collection: DataCollection = DataCollection.SENTINEL2_L2A, + **kwargs: Any, + ): """ :param sen2cor_classification: "SCL" (scene classification), "CLD" (cloud probability) or "SNW" (snow probability) masks to be retrieved. Also, a list of their combination (e.g. ["SCL","CLD"]) - :type sen2cor_classification: str or [str] :param kwargs: Additional arguments that will be passed to the `SentinelHubInputTask` """ # definition of possible types and target features @@ -697,7 +732,7 @@ def __init__(self, sen2cor_classification, data_collection=DataCollection.SENTIN if data_collection != DataCollection.SENTINEL2_L2A: raise ValueError("Sen2Cor classification layers are only available on Sentinel-2 L2A data.") - features = [(classification_types[s2c], s2c) for s2c in sen2cor_classification] + features: List[Tuple[FeatureType, str]] = [(classification_types[s2c], s2c) for s2c in sen2cor_classification] super().__init__(additional_data=features, data_collection=data_collection, **kwargs) From 966ab9329a779b965dd6ac4781aeb43700a7dcbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Tue, 15 Nov 2022 09:35:59 +0100 Subject: [PATCH 07/14] Add custom timestamp filtration (#500) * Adds option for the user to provide custom timestamp filtration * Add docstring with typo to add a collaborator * Fix intended typo to add co-author Co-authored-by: Colin Moldenhauer * generalize terms in docstring Co-authored-by: Colin Moldenhauer --- io/eolearn/io/sentinelhub_process.py | 17 ++++++++++++++- io/eolearn/tests/test_sentinelhub_process.py | 22 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/io/eolearn/io/sentinelhub_process.py b/io/eolearn/io/sentinelhub_process.py index 853604eee..185928443 100644 --- a/io/eolearn/io/sentinelhub_process.py +++ b/io/eolearn/io/sentinelhub_process.py @@ -222,6 +222,7 @@ def __init__( downsampling: Optional[ResamplingType] = None, aux_request_args: Optional[dict] = None, session_loader: Optional[Callable[[], SentinelHubSession]] = None, + timestamp_filter: Callable[[List[dt.datetime], dt.timedelta], List[dt.datetime]] = filter_times, ): """ :param features: Features to construct from the evalscript. @@ -242,6 +243,8 @@ def __init__( :param aux_request_args: a dictionary with auxiliary information for the input_data part of the SH request :param session_loader: A callable that returns a valid SentinelHubSession, used for session sharing. Creates a new session if set to `None`, which should be avoided in large scale parallelization. + :param timestamp_filter: A function that performs the final filtering of timestamps, usually to remove multiple + occurrences within the time_difference window. Check `get_available_timestamps` for more info. """ super().__init__( data_collection=data_collection, @@ -267,6 +270,7 @@ def __init__( self.maxcc = maxcc self.time_difference = time_difference or dt.timedelta(seconds=1) + self.timestamp_filter = timestamp_filter self.mosaicking_order = None if mosaicking_order is None else MosaickingOrder(mosaicking_order) self.aux_request_args = aux_request_args @@ -305,6 +309,7 @@ def _get_timestamp(self, time_interval, bbox): return get_available_timestamps( bbox=bbox, time_interval=time_interval, + timestamp_filter=self.timestamp_filter, data_collection=self.data_collection, maxcc=self.maxcc, time_difference=self.time_difference, @@ -413,6 +418,7 @@ def __init__( downsampling: Optional[ResamplingType] = None, aux_request_args: Optional[dict] = None, session_loader: Optional[Callable[[], SentinelHubSession]] = None, + timestamp_filter: Callable[[List[dt.datetime], dt.timedelta], List[dt.datetime]] = filter_times, ): """ :param data_collection: Source of requested satellite data. @@ -438,6 +444,8 @@ def __init__( :param aux_request_args: a dictionary with auxiliary information for the input_data part of the SH request :param session_loader: A callable that returns a valid SentinelHubSession, used for session sharing. Creates a new session if set to `None`, which should be avoided in large scale parallelization. + :param timestamp_filter: A callable that performs the final filtering of timestamps, usually to remove multiple + occurrences within the time_difference window. Check `get_available_timestamps` for more info. """ super().__init__( data_collection=data_collection, @@ -453,6 +461,7 @@ def __init__( self.evalscript = evalscript self.maxcc = maxcc self.time_difference = time_difference or dt.timedelta(seconds=1) + self.timestamp_filter = timestamp_filter self.single_scene = single_scene self.bands_dtype = bands_dtype self.mosaicking_order = None if mosaicking_order is None else MosaickingOrder(mosaicking_order) @@ -547,6 +556,7 @@ def _get_timestamp(self, time_interval, bbox): return get_available_timestamps( bbox=bbox, time_interval=time_interval, + timestamp_filter=self.timestamp_filter, data_collection=self.data_collection, maxcc=self.maxcc, time_difference=self.time_difference, @@ -742,6 +752,7 @@ def get_available_timestamps( *, time_interval: Optional[Tuple[dt.datetime, dt.datetime]] = None, time_difference: dt.timedelta = dt.timedelta(seconds=-1), # noqa: B008 + timestamp_filter: Callable[[List[dt.datetime], dt.timedelta], List[dt.datetime]] = filter_times, maxcc: Optional[float] = None, config: Optional[SHConfig] = None, ) -> List[dt.datetime]: @@ -751,6 +762,10 @@ def get_available_timestamps( :param data_collection: A data collection for which to find available timestamps. :param time_interval: A time interval from which to provide the timestamps. :param time_difference: Minimum allowed time difference, used when filtering dates. + :param timestamp_filter: A function that performs the final filtering of timestamps, usually to remove multiple + occurrences within the time_difference window. The filtration is performed after all suitable timestamps for + the given region are obtained (with maxcc filtering already done by SH). By default only keeps the oldest + timestamp when multiple occur within `time_difference`. :param maxcc: Maximum cloud coverage filter from interval [0, 1], default is None. :param config: A configuration object. :return: A list of timestamps of available observations. @@ -773,4 +788,4 @@ def get_available_timestamps( ) all_timestamps = search_iterator.get_timestamps() - return filter_times(all_timestamps, time_difference) + return timestamp_filter(all_timestamps, time_difference) diff --git a/io/eolearn/tests/test_sentinelhub_process.py b/io/eolearn/tests/test_sentinelhub_process.py index dc8c421c7..322f469d7 100644 --- a/io/eolearn/tests/test_sentinelhub_process.py +++ b/io/eolearn/tests/test_sentinelhub_process.py @@ -524,6 +524,28 @@ def test_get_available_timestamps_with_missing_data_collection_service_url(self) assert len(timestamps) == 4 assert all(timestamp.tzinfo is not None for timestamp in timestamps) + def test_get_available_timestamps_custom_filtration(self): + """Checks that the custom filtration works as intended.""" + timestamps1 = get_available_timestamps( + bbox=self.bbox, + config=SHConfig(), + data_collection=DataCollection.SENTINEL2_L1C, + time_interval=self.time_interval, + timestamp_filter=lambda stamps, diff: stamps[:3], + ) + + assert len(timestamps1) == 3 + + timestamps2 = get_available_timestamps( + bbox=self.bbox, + config=SHConfig(), + data_collection=DataCollection.SENTINEL2_L1C, + time_interval=self.time_interval, + timestamp_filter=lambda stamps, diff: stamps[:5], + ) + + assert len(timestamps2) == 5 + def test_no_data_input_task_request(self): task = SentinelHubInputTask( bands_feature=(FeatureType.DATA, "BANDS"), From ab34e5e9c018fa47b50606b49053d0420dd04fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Wed, 16 Nov 2022 11:34:39 +0100 Subject: [PATCH 08/14] Fix SpatialResizeTask docstring and do slight refactoring. (#501) * Fix SpatialResizeTask docstring and do slight refactoring. * fix faulty error message --- .../eolearn/features/feature_manipulation.py | 45 ++++++++----------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/features/eolearn/features/feature_manipulation.py b/features/eolearn/features/feature_manipulation.py index 872a411d3..b09de0b41 100644 --- a/features/eolearn/features/feature_manipulation.py +++ b/features/eolearn/features/feature_manipulation.py @@ -15,14 +15,15 @@ import datetime as dt import logging from functools import partial -from typing import Any, Callable, Dict, Iterable, List, Tuple, Union +from typing import Any, Callable, Iterable, List, Tuple, Union import numpy as np from geopandas import GeoDataFrame -from sentinelhub import BBox, bbox_to_dimensions +from sentinelhub import bbox_to_dimensions from eolearn.core import EOPatch, EOTask, FeatureType, FeatureTypeSet, MapFeatureTask +from eolearn.core.utils.parsing import FeaturesSpecification from .utils import ResizeLib, ResizeMethod, ResizeParam, spatially_resize_image @@ -259,48 +260,40 @@ def map_method(self, feature: np.ndarray, slope: float, intercept: float) -> np. class SpatialResizeTask(EOTask): """Resizes the specified spatial features of EOPatch.""" - def _process_resize_parameters( - self, resize_parameters: Tuple[ResizeParam, Tuple[float, float]], bbox: BBox - ) -> Dict[str, Tuple[float, float]]: - resize_type, resize_type_param = resize_parameters - resize_type = ResizeParam(resize_type) - - if resize_type == ResizeParam.RESOLUTION: - new_size = bbox_to_dimensions(bbox, resize_type_param) - return {ResizeParam.NEW_SIZE.value: new_size} - - return {resize_type.value: resize_type_param} - def __init__( self, *, resize_parameters: Tuple[ResizeParam, Tuple[float, float]], - features: Any = ..., + features: FeaturesSpecification = ..., resize_method: ResizeMethod = ResizeMethod.LINEAR, resize_library: ResizeLib = ResizeLib.PIL, ): """ - :param features: The specification of feature for which to perform resizing. Must be supported by the - :class:`FeatureParser`. Features can be renamed, see `FeatureParser` - documentation. - :param new_size: New size of the data (height, width) - :param scale_factors: Factors (f_height, f_width) by which to resize the image + :param features: Which features to resize. Supports new names for features. + :param resize_parameters: Instructions on how to perform the resizing process. For example use: + `['new_size', [500, 1000]]` to resize the data to size (500, 1000), + `['resolution', [10, 20]]` for changing resolution from 10 to 20, + `['scale_factors', [3, 3]]` to make the data three times larger. :param resize_method: Interpolation method used for resizing. :param resize_library: Which Python library to use for resizing. Default is PIL, as it supports all dtypes and features anti-aliasing. For cases where execution speed is crucial one can use CV2. """ self.features = features - self.resize_parameters = resize_parameters + self.parameter_kind = ResizeParam(resize_parameters[0]) + self.parameter_values = resize_parameters[1] self.resize_function = partial( - spatially_resize_image, - resize_method=resize_method, - resize_library=resize_library, + spatially_resize_image, resize_method=resize_method, resize_library=resize_library ) def execute(self, eopatch: EOPatch) -> EOPatch: - resize_params = self._process_resize_parameters(self.resize_parameters, eopatch.bbox) + if self.parameter_kind == ResizeParam.RESOLUTION: + new_size = bbox_to_dimensions(eopatch.bbox, self.parameter_values) + resize_fun_kwargs = {ResizeParam.NEW_SIZE.value: new_size} + else: + resize_fun_kwargs = {self.parameter_kind.value: self.parameter_values} + for ftype, fname, new_name in self.parse_renamed_features(self.features, eopatch=eopatch): if ftype.is_spatial() and ftype.is_raster(): - eopatch[ftype, new_name] = self.resize_function(eopatch[ftype, fname], **resize_params) + eopatch[ftype, new_name] = self.resize_function(eopatch[ftype, fname], **resize_fun_kwargs) return eopatch From ea971d651d1613726bbfef0f7db79c4843c947bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Thu, 17 Nov 2022 16:10:12 +0100 Subject: [PATCH 09/14] Small refactor fo sentinelhub IO tasks (#502) * avoid mutating method * remove awkward timestamp comparisson * remove disfunctional defaults * refine type information --- io/eolearn/io/sentinelhub_process.py | 96 +++++++++++----------------- 1 file changed, 39 insertions(+), 57 deletions(-) diff --git a/io/eolearn/io/sentinelhub_process.py b/io/eolearn/io/sentinelhub_process.py index 185928443..97c08847e 100644 --- a/io/eolearn/io/sentinelhub_process.py +++ b/io/eolearn/io/sentinelhub_process.py @@ -38,7 +38,7 @@ from sentinelhub.type_utils import RawTimeIntervalType from eolearn.core import EOPatch, EOTask, FeatureType, FeatureTypeSet -from eolearn.core.utils.parsing import FeatureSpec, FeaturesSpecification +from eolearn.core.utils.parsing import FeatureRenameSpec, FeatureSpec, FeaturesSpecification from eolearn.core.utils.types import Literal LOGGER = logging.getLogger(__name__) @@ -97,26 +97,26 @@ def execute( eopatch = eopatch or EOPatch() - self._check_and_set_eopatch_bbox(bbox, eopatch) + eopatch.bbox = self._extract_bbox(bbox, eopatch) size_x, size_y = self._get_size(eopatch) if time_interval: time_interval = parse_time_interval(time_interval) timestamp = self._get_timestamp(time_interval, eopatch.bbox) + timestamp = [time_point.replace(tzinfo=None) for time_point in timestamp] elif self.data_collection.is_timeless: - timestamp = None + timestamp = None # should this be [] to match next branch in case of a fresh eopatch? else: timestamp = eopatch.timestamp if timestamp is not None: - eop_timestamp = [time_point.replace(tzinfo=None) for time_point in timestamp] - if eopatch.timestamp: - self.check_timestamp_difference(eop_timestamp, eopatch.timestamp) - else: - eopatch.timestamp = eop_timestamp + if not eopatch.timestamp: + eopatch.timestamp = timestamp + elif timestamp != eopatch.timestamp: + raise ValueError("Trying to write data to an existing EOPatch with a different timestamp.") - requests = self._build_requests(eopatch.bbox, size_x, size_y, timestamp, time_interval, geometry) - requests = [request.download_list[0] for request in requests] + sh_requests = self._build_requests(eopatch.bbox, size_x, size_y, timestamp, time_interval, geometry) + requests = [request.download_list[0] for request in sh_requests] LOGGER.debug("Downloading %d requests of type %s", len(requests), str(self.data_collection)) session = None if self.session_loader is None else self.session_loader() @@ -158,45 +158,33 @@ def _add_meta_info(self, eopatch): eopatch.meta_info["time_difference"] = self.time_difference.total_seconds() @staticmethod - def _check_and_set_eopatch_bbox(bbox, eopatch): + def _extract_bbox(bbox: Optional[BBox], eopatch: EOPatch) -> BBox: if eopatch.bbox is None: if bbox is None: raise ValueError("Either the eopatch or the task must provide valid bbox.") - eopatch.bbox = bbox - return + return bbox if bbox is None or eopatch.bbox == bbox: - return + return eopatch.bbox raise ValueError("Either the eopatch or the task must provide bbox, or they must be the same.") - @staticmethod - def check_timestamp_difference(timestamp1, timestamp2): - """Raises an error if the two timestamps are not the same""" - error_msg = "Trying to write data to an existing EOPatch with a different timestamp." - if len(timestamp1) != len(timestamp2): - raise ValueError(error_msg) - - for ts1, ts2 in zip(timestamp1, timestamp2): - if ts1 != ts2: - raise ValueError(error_msg) - def _extract_data(self, eopatch, images, shape): """Extract data from the received images and assign them to eopatch features""" raise NotImplementedError("The _extract_data method should be implemented by the subclass.") def _build_requests( self, - bbox: BBox, + bbox: Optional[BBox], size_x: int, size_y: int, timestamp: Optional[List[dt.datetime]], - time_interval: Optional[Tuple[dt.datetime, dt.datetime]], - geometry: Geometry, + time_interval: Optional[RawTimeIntervalType], + geometry: Optional[Geometry], ) -> List[SentinelHubRequest]: """Build requests""" raise NotImplementedError("The _build_requests method should be implemented by the subclass.") - def _get_timestamp(self, time_interval: Optional[Tuple[dt.datetime, dt.datetime]], bbox: BBox) -> List[dt.datetime]: + def _get_timestamp(self, time_interval: Optional[RawTimeIntervalType], bbox: BBox) -> List[dt.datetime]: """Get the timestamp array needed as a parameter for downloading the images""" raise NotImplementedError("The _get_timestamp method should be implemented by the subclass.") @@ -207,9 +195,9 @@ class SentinelHubEvalscriptTask(SentinelHubInputBaseTask): # pylint: disable=too-many-arguments def __init__( self, - features: Optional[FeaturesSpecification] = None, - evalscript: Optional[str] = None, - data_collection: Optional[DataCollection] = None, + features: FeaturesSpecification, + evalscript: str, + data_collection: DataCollection, size: Optional[Tuple[int, int]] = None, resolution: Optional[Union[float, Tuple[float, float]]] = None, maxcc: Optional[float] = None, @@ -260,9 +248,6 @@ def __init__( self.features = self._parse_and_validate_features(features) self.responses = self._create_response_objects() - - if not evalscript: - raise ValueError("evalscript parameter must not be missing/empty") self.evalscript = evalscript if maxcc and isinstance(maxcc, (int, float)) and (maxcc < 0 or maxcc > 1): @@ -274,10 +259,7 @@ def __init__( self.mosaicking_order = None if mosaicking_order is None else MosaickingOrder(mosaicking_order) self.aux_request_args = aux_request_args - def _parse_and_validate_features(self, features): - if not features: - raise ValueError("features must be defined") - + def _parse_and_validate_features(self, features: FeaturesSpecification) -> List[FeatureRenameSpec]: allowed_features = FeatureTypeSet.RASTER_TYPES.union({FeatureType.META_INFO}) _features = self.parse_renamed_features(features, allowed_feature_types=allowed_features) @@ -301,7 +283,7 @@ def _create_response_objects(self): return responses - def _get_timestamp(self, time_interval, bbox): + def _get_timestamp(self, time_interval: Optional[RawTimeIntervalType], bbox: BBox) -> List[dt.datetime]: """Get the timestamp array needed as a parameter for downloading the images""" if any(feat_type.is_timeless() for feat_type, _, _ in self.features if feat_type.is_raster()): return [] @@ -318,12 +300,12 @@ def _get_timestamp(self, time_interval, bbox): def _build_requests( self, - bbox: BBox, + bbox: Optional[BBox], size_x: int, size_y: int, - timestamp: List[dt.datetime], - time_interval: Optional[Tuple[dt.datetime, dt.datetime]], - geometry: Geometry, + timestamp: Optional[List[dt.datetime]], + time_interval: Optional[RawTimeIntervalType], + geometry: Optional[Geometry], ): """Defines request timestamps and builds requests. In case `timestamp` is either `None` or an empty list it still has to create at least one request in order to obtain back number of bands of responses.""" @@ -399,7 +381,7 @@ class SentinelHubInputTask(SentinelHubInputBaseTask): # pylint: disable=too-many-locals def __init__( self, - data_collection: Optional[DataCollection] = None, + data_collection: DataCollection, size: Optional[Tuple[int, int]] = None, resolution: Optional[Union[float, Tuple[float, float]]] = None, bands_feature: Optional[Tuple[FeatureType, str]] = None, @@ -411,7 +393,7 @@ def __init__( cache_folder: Optional[str] = None, config: Optional[SHConfig] = None, max_threads: Optional[int] = None, - bands_dtype: Union[np.dtype, type] = None, + bands_dtype: Union[None, np.dtype, type] = None, single_scene: bool = False, mosaicking_order: Optional[Union[str, MosaickingOrder]] = None, upsampling: Optional[ResamplingType] = None, @@ -548,7 +530,7 @@ def generate_evalscript(self) -> str: return evalscript - def _get_timestamp(self, time_interval, bbox): + def _get_timestamp(self, time_interval: Optional[RawTimeIntervalType], bbox: BBox) -> List[dt.datetime]: """Get the timestamp array needed as a parameter for downloading the images""" if self.single_scene: return [time_interval[0]] @@ -565,24 +547,24 @@ def _get_timestamp(self, time_interval, bbox): def _build_requests( self, - bbox: BBox, + bbox: Optional[BBox], size_x: int, size_y: int, timestamp: Optional[List[dt.datetime]], - time_interval: Optional[Tuple[dt.datetime, dt.datetime]], - geometry: Geometry, + time_interval: Optional[RawTimeIntervalType], + geometry: Optional[Geometry], ) -> List[SentinelHubRequest]: """Build requests""" if timestamp is None: - dates = [None] + intervals: List[Optional[RawTimeIntervalType]] = [None] elif self.single_scene: - dates = [parse_time_interval(time_interval)] + intervals = [parse_time_interval(time_interval)] else: - dates = [(date - self.time_difference, date + self.time_difference) for date in timestamp] + intervals = [(date - self.time_difference, date + self.time_difference) for date in timestamp] - return [self._create_sh_request(date1, date2, bbox, size_x, size_y, geometry) for date1, date2 in dates] + return [self._create_sh_request(time_interval, bbox, size_x, size_y, geometry) for time_interval in intervals] - def _create_sh_request(self, date_from, date_to, bbox, size_x, size_y, geometry): + def _create_sh_request(self, time_interval, bbox, size_x, size_y, geometry): """Create an instance of SentinelHubRequest""" responses = [ SentinelHubRequest.output_response(band.name, MimeType.TIFF) @@ -594,7 +576,7 @@ def _create_sh_request(self, date_from, date_to, bbox, size_x, size_y, geometry) input_data=[ SentinelHubRequest.input_data( data_collection=self.data_collection, - time_interval=(date_from, date_to), + time_interval=time_interval, mosaicking_order=self.mosaicking_order, maxcc=self.maxcc, upsampling=self.upsampling, @@ -750,7 +732,7 @@ def get_available_timestamps( bbox: BBox, data_collection: DataCollection, *, - time_interval: Optional[Tuple[dt.datetime, dt.datetime]] = None, + time_interval: Optional[RawTimeIntervalType] = None, time_difference: dt.timedelta = dt.timedelta(seconds=-1), # noqa: B008 timestamp_filter: Callable[[List[dt.datetime], dt.timedelta], List[dt.datetime]] = filter_times, maxcc: Optional[float] = None, From 0c9d5cd514b6c8ba2a365b2c7a3b9740ad3bfbe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Mon, 21 Nov 2022 08:42:55 +0100 Subject: [PATCH 10/14] Feat/use sh testing utils (#504) * switch blob tests to sh-py testing utils * refactor other tests in eolearn.features * try adjusting delta for Kriging task --- features/eolearn/tests/conftest.py | 2 + features/eolearn/tests/test_blob.py | 51 +++++---- features/eolearn/tests/test_haralick.py | 40 ++----- features/eolearn/tests/test_hog.py | 28 ++--- features/eolearn/tests/test_interpolation.py | 102 +++++------------- .../tests/test_local_binary_pattern.py | 35 +++--- .../tests/test_radiometric_normalization.py | 65 +++-------- 7 files changed, 103 insertions(+), 220 deletions(-) diff --git a/features/eolearn/tests/conftest.py b/features/eolearn/tests/conftest.py index d2c2c5776..e319f4dbb 100644 --- a/features/eolearn/tests/conftest.py +++ b/features/eolearn/tests/conftest.py @@ -8,6 +8,8 @@ from eolearn.core import EOPatch +pytest.register_assert_rewrite("sentinelhub.testing_utils") # makes asserts in helper functions work with pytest + EXAMPLE_DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", "example_data") EXAMPLE_EOPATCH_PATH = os.path.join(EXAMPLE_DATA_PATH, "TestEOPatch") diff --git a/features/eolearn/tests/test_blob.py b/features/eolearn/tests/test_blob.py index 7e294fe0e..af3feee15 100644 --- a/features/eolearn/tests/test_blob.py +++ b/features/eolearn/tests/test_blob.py @@ -11,51 +11,50 @@ import copy import sys -import numpy as np import pytest -from numpy.testing import assert_array_equal from pytest import approx from skimage.feature import blob_dog +from sentinelhub.testing_utils import test_numpy_data + from eolearn.core import FeatureType -from eolearn.core.eodata_io import FeatureIO from eolearn.features import BlobTask, DoGBlobTask, DoHBlobTask, LoGBlobTask FEATURE = (FeatureType.DATA, "NDVI", "blob") +BLOB_FEATURE = (FeatureType.DATA, "blob") -def test_blob_feature(small_ndvi_eopatch): +def test_dog_blob_task(small_ndvi_eopatch): eopatch = small_ndvi_eopatch BlobTask(FEATURE, blob_dog, sigma_ratio=1.6, min_sigma=1, max_sigma=30, overlap=0.5, threshold=0)(eopatch) DoGBlobTask((FeatureType.DATA, "NDVI", "blob_dog"), threshold=0)(eopatch) - assert eopatch.data["blob"] == approx( - eopatch.data["blob_dog"] - ), "DoG derived class result not equal to base class result" + assert eopatch[BLOB_FEATURE] == approx(eopatch.data["blob_dog"]) BLOB_TESTS = [ - [DoGBlobTask(FEATURE, threshold=0), 0.0, 37.9625, 0.0854, 0.0], + (DoGBlobTask(FEATURE, threshold=0), {"exp_min": 0.0, "exp_max": 37.9625, "exp_mean": 0.08545, "exp_median": 0.0}), ] if sys.version_info >= (3, 8): # For Python 3.7 scikit-image returns less accurate result for this test - BLOB_TESTS.append([DoHBlobTask(FEATURE, num_sigma=5, threshold=0), 0.0, 21.9203, 0.05807, 0.0]) - BLOB_TESTS.append([LoGBlobTask(FEATURE, log_scale=True, threshold=0), 0, 42.4264, 0.0977, 0.0]) - - -@pytest.mark.parametrize("task, expected_min, expected_max, expected_mean, expected_median", BLOB_TESTS) -def test_blob(small_ndvi_eopatch, task, expected_min, expected_max, expected_mean, expected_median): + BLOB_TESTS.extend( + [ + ( + DoHBlobTask(FEATURE, num_sigma=5, threshold=0), + {"exp_min": 0.0, "exp_max": 21.9203, "exp_mean": 0.05807, "exp_median": 0.0}, + ), + ( + LoGBlobTask(FEATURE, log_scale=True, threshold=0), + {"exp_min": 0, "exp_max": 42.4264, "exp_mean": 0.09767, "exp_median": 0.0}, + ), + ] + ) + + +@pytest.mark.parametrize("task, expected_statistics", BLOB_TESTS) +def test_blob_task(small_ndvi_eopatch, task, expected_statistics): eopatch = copy.deepcopy(small_ndvi_eopatch) task.execute(eopatch) - # Test that no other features were modified - for feature, value in small_ndvi_eopatch.data.items(): - if isinstance(value, FeatureIO): - value = value.load() - assert_array_equal(value, eopatch.data[feature], err_msg=f"EOPatch data feature '{feature}' has changed") - - delta = 1e-4 + test_numpy_data(eopatch[BLOB_FEATURE], exp_shape=(10, 20, 20, 1), **expected_statistics, delta=1e-4) - blob = eopatch.data[FEATURE[-1]] - assert np.min(blob) == approx(expected_min, abs=delta) - assert np.max(blob) == approx(expected_max, abs=delta) - assert np.mean(blob) == approx(expected_mean, abs=delta) - assert np.median(blob) == approx(expected_median, abs=delta) + del eopatch[BLOB_FEATURE] + assert small_ndvi_eopatch == eopatch, "Other features of the EOPatch were affected." diff --git a/features/eolearn/tests/test_haralick.py b/features/eolearn/tests/test_haralick.py index b03cdfce7..de1515ed7 100644 --- a/features/eolearn/tests/test_haralick.py +++ b/features/eolearn/tests/test_haralick.py @@ -10,56 +10,38 @@ import numpy as np import pytest -from numpy.testing import assert_array_equal -from pytest import approx + +from sentinelhub.testing_utils import test_numpy_data from eolearn.core import FeatureType -from eolearn.core.eodata_io import FeatureIO from eolearn.features import HaralickTask FEATURE = (FeatureType.DATA, "NDVI", "haralick") +OUTPUT_FEATURE = (FeatureType.DATA, "haralick") @pytest.mark.parametrize( - "task, expected_min, expected_max, expected_mean, expected_median", + "task, expected_statistics", ( [ HaralickTask(FEATURE, texture_feature="contrast", angle=0, levels=255, window_size=3), - 3.5, - 9079.0, - 965.8295, - 628.5833, + {"exp_min": 3.5, "exp_max": 9079.0, "exp_mean": 965.8295, "exp_median": 628.5833}, ], [ HaralickTask(FEATURE, texture_feature="sum_of_square_variance", angle=np.pi / 2, levels=8, window_size=5), - 0.96899, - 48.7815, - 23.0229, - 23.8987, + {"exp_min": 0.96899, "exp_max": 48.7815, "exp_mean": 23.0229, "exp_median": 23.8987}, ], [ HaralickTask(FEATURE, texture_feature="sum_entropy", angle=-np.pi / 2, levels=8, window_size=7), - 0, - 1.7463, - 0.5657, - 0.5055, + {"exp_min": 0, "exp_max": 1.7463, "exp_mean": 0.5657, "exp_median": 0.50558}, ], ), ) -def test_haralick(small_ndvi_eopatch, task, expected_min, expected_max, expected_mean, expected_median): +def test_haralick(small_ndvi_eopatch, task, expected_statistics): eopatch = copy.deepcopy(small_ndvi_eopatch) task.execute(eopatch) - # Test that no other features were modified - for feature, value in small_ndvi_eopatch.data.items(): - if isinstance(value, FeatureIO): - value = value.load() - assert_array_equal(value, eopatch.data[feature], err_msg=f"EOPatch data feature '{feature}' has changed") - - delta = 1e-4 + test_numpy_data(eopatch[OUTPUT_FEATURE], exp_shape=(10, 20, 20, 1), **expected_statistics, delta=1e-4) - haralick = eopatch.data["haralick"] - assert np.min(haralick) == approx(expected_min, abs=delta) - assert np.max(haralick) == approx(expected_max, abs=delta) - assert np.mean(haralick) == approx(expected_mean, abs=delta) - assert np.median(haralick) == approx(expected_median, abs=delta) + del eopatch[OUTPUT_FEATURE] + assert small_ndvi_eopatch == eopatch, "Other features of the EOPatch were affected." diff --git a/features/eolearn/tests/test_hog.py b/features/eolearn/tests/test_hog.py index 7b80fe36c..2f215b9f6 100644 --- a/features/eolearn/tests/test_hog.py +++ b/features/eolearn/tests/test_hog.py @@ -8,12 +8,9 @@ """ import copy -import numpy as np -from numpy.testing import assert_array_equal -from pytest import approx +from sentinelhub.testing_utils import test_numpy_data from eolearn.core import FeatureType -from eolearn.core.eodata_io import FeatureIO from eolearn.features import HOGTask @@ -30,19 +27,12 @@ def test_hog(small_ndvi_eopatch): eopatch = copy.deepcopy(small_ndvi_eopatch) task.execute(eopatch) - # Test that no other features were modified - for feature, value in small_ndvi_eopatch.data.items(): - if isinstance(value, FeatureIO): - value = value.load() - assert_array_equal(value, eopatch.data[feature], err_msg=f"EOPatch data feature '{feature}' has changed") - - delta = 1e-4 - for feature, expected_min, expected_max, expected_mean, expected_median in [ - ("hog", 0.0, 0.5567, 0.0931, 0.0), - ("hog_visu", 0.0, 0.3241, 0.0105, 0.0), + for feature_name, expected_statistics in [ + ("hog", {"exp_min": 0.0, "exp_max": 0.5567, "exp_mean": 0.09309, "exp_median": 0.0}), + ("hog_visu", {"exp_min": 0.0, "exp_max": 0.3241, "exp_mean": 0.010537, "exp_median": 0.0}), ]: - hog = eopatch.data[feature] - assert np.min(hog) == approx(expected_min, abs=delta) - assert np.max(hog) == approx(expected_max, abs=delta) - assert np.mean(hog) == approx(expected_mean, abs=delta) - assert np.median(hog) == approx(expected_median, abs=delta) + test_numpy_data(eopatch.data[feature_name], **expected_statistics, delta=1e-4) + + del eopatch[(FeatureType.DATA, "hog")] + del eopatch[(FeatureType.DATA, "hog_visu")] + assert small_ndvi_eopatch == eopatch, "Other features of the EOPatch were affected." diff --git a/features/eolearn/tests/test_interpolation.py b/features/eolearn/tests/test_interpolation.py index f33ad509e..17f032803 100644 --- a/features/eolearn/tests/test_interpolation.py +++ b/features/eolearn/tests/test_interpolation.py @@ -11,11 +11,12 @@ """ import dataclasses from datetime import datetime -from typing import Optional +from typing import Dict, Optional import numpy as np import pytest -from pytest import approx + +from sentinelhub.testing_utils import test_numpy_data from eolearn.core import EOPatch, EOTask, FeatureType from eolearn.features import ( @@ -53,10 +54,7 @@ class InterpolationTestCase: name: str task: EOTask result_len: int - img_min: float - img_max: float - img_mean: float - img_median: float + expected_statistics: Dict[str, float] nan_replace: Optional[float] = None def execute(self, eopatch): @@ -82,10 +80,7 @@ def execute(self, eopatch): unknown_value=10, ), result_len=68, - img_min=0.0, - img_max=0.82836, - img_mean=0.51187, - img_median=0.57889, + expected_statistics=dict(exp_min=0.0, exp_max=0.82836, exp_mean=0.51187, exp_median=0.57889), ), InterpolationTestCase( "linear-p", @@ -97,10 +92,7 @@ def execute(self, eopatch): interpolate_pixel_wise=True, ), result_len=68, - img_min=0.0, - img_max=0.82836, - img_mean=0.51187, - img_median=0.57889, + expected_statistics=dict(exp_min=0.0, exp_max=0.82836, exp_mean=0.51187, exp_median=0.57889), ), InterpolationTestCase( "linear_change_timescale", @@ -112,10 +104,7 @@ def execute(self, eopatch): scale_time=1, ), result_len=68, - img_min=0.0, - img_max=0.82836, - img_mean=0.51187, - img_median=0.57889, + expected_statistics=dict(exp_min=0.0, exp_max=0.82836, exp_mean=0.51187, exp_median=0.57889), ), InterpolationTestCase( "cubic", @@ -128,10 +117,7 @@ def execute(self, eopatch): bounds_error=False, ), result_len=69, - img_min=0.0, - img_max=5.0, - img_mean=1.3532, - img_median=0.638732, + expected_statistics=dict(exp_min=0.0, exp_max=5.0, exp_mean=1.3532, exp_median=0.638732), ), InterpolationTestCase( "spline", @@ -145,10 +131,7 @@ def execute(self, eopatch): unknown_value=0, ), result_len=147, - img_min=-0.17814, - img_max=1.0, - img_mean=0.49738, - img_median=0.556853, + expected_statistics=dict(exp_min=-0.1781458, exp_max=1.0, exp_mean=0.49738, exp_median=0.556853), ), InterpolationTestCase( "bspline", @@ -160,10 +143,7 @@ def execute(self, eopatch): spline_degree=5, ), result_len=1, - img_min=-0.0163, - img_max=0.62323, - img_mean=0.319117, - img_median=0.32588, + expected_statistics=dict(exp_min=-0.0162962, exp_max=0.62323, exp_mean=0.319117, exp_median=0.3258836), ), InterpolationTestCase( "bspline-p", @@ -176,10 +156,7 @@ def execute(self, eopatch): interpolate_pixel_wise=True, ), result_len=1, - img_min=-0.0163, - img_max=0.62323, - img_mean=0.319117, - img_median=0.32588, + expected_statistics=dict(exp_min=-0.0162962, exp_max=0.62323, exp_mean=0.319117, exp_median=0.3258836), ), InterpolationTestCase( "akima", @@ -187,10 +164,7 @@ def execute(self, eopatch): (FeatureType.DATA, "NDVI"), unknown_value=0, mask_feature=(FeatureType.MASK, "IS_VALID") ), result_len=68, - img_min=-0.091035, - img_max=0.8283603, - img_mean=0.51427454, - img_median=0.59095883, + expected_statistics=dict(exp_min=-0.091035, exp_max=0.8283603, exp_mean=0.51427454, exp_median=0.59095883), ), InterpolationTestCase( "kriging interpolation", @@ -198,10 +172,7 @@ def execute(self, eopatch): (FeatureType.DATA, "NDVI"), result_interval=(-10, 10), resample_range=("2017-01-01", "2018-01-01", 10) ), result_len=37, - img_min=-0.18389, - img_max=0.5995388, - img_mean=0.35485545, - img_median=0.37279952, + expected_statistics=dict(exp_min=-0.183885, exp_max=0.5995388, exp_mean=0.35485545, exp_median=0.37279952), ), InterpolationTestCase( "nearest resample", @@ -209,10 +180,7 @@ def execute(self, eopatch): (FeatureType.DATA, "NDVI"), result_interval=(0.0, 1.0), resample_range=("2016-01-01", "2018-01-01", 5) ), result_len=147, - img_min=-0.2, - img_max=0.8283603, - img_mean=0.32318678, - img_median=0.2794411, + expected_statistics=dict(exp_min=-0.2, exp_max=0.8283603, exp_mean=0.32318678, exp_median=0.2794411), nan_replace=-0.2, ), InterpolationTestCase( @@ -221,10 +189,7 @@ def execute(self, eopatch): (FeatureType.DATA, "NDVI"), result_interval=(0.0, 1.0), resample_range=("2016-01-01", "2018-01-01", 5) ), result_len=147, - img_min=-0.2, - img_max=0.82643485, - img_mean=0.32218185, - img_median=0.29093677, + expected_statistics=dict(exp_min=-0.2, exp_max=0.82643485, exp_mean=0.32218185, exp_median=0.29093677), nan_replace=-0.2, ), InterpolationTestCase( @@ -236,10 +201,7 @@ def execute(self, eopatch): unknown_value=5, ), result_len=69, - img_min=-0.2, - img_max=5.0, - img_mean=1.209852, - img_median=0.40995836, + expected_statistics=dict(exp_min=-0.2, exp_max=5.0, exp_mean=1.209852, exp_median=0.40995836), nan_replace=-0.2, ), InterpolationTestCase( @@ -252,10 +214,7 @@ def execute(self, eopatch): resample_range=("2015-09-01", "2016-01-01", "2016-07-01", "2017-01-01", "2017-07-01"), ), result_len=5, - img_min=-0.0252167, - img_max=0.816656, - img_mean=0.49966, - img_median=0.533415, + expected_statistics=dict(exp_min=-0.0252167, exp_max=0.816656, exp_mean=0.49966, exp_median=0.533415), ), InterpolationTestCase( "linear with bands and multiple masks", @@ -270,10 +229,7 @@ def execute(self, eopatch): ], ), result_len=68, - img_min=0.0003, - img_max=10.0, - img_mean=0.132176, - img_median=0.086, + expected_statistics=dict(exp_min=0.0003, exp_max=10.0, exp_mean=0.132176, exp_median=0.086), ), ] @@ -295,10 +251,7 @@ def execute(self, eopatch): ], ), result_len=69, - img_min=0.0, - img_max=5.0, - img_mean=1.3592644, - img_median=0.6174331, + expected_statistics=dict(exp_min=0.0, exp_max=5.0, exp_mean=1.3592644, exp_median=0.6174331), ), InterpolationTestCase( "cubic_copy_fail", @@ -316,17 +269,15 @@ def execute(self, eopatch): ], ), result_len=69, - img_min=0.0, - img_max=5.0, - img_mean=1.3592644, - img_median=0.6174331, + expected_statistics=dict(exp_min=0.0, exp_max=5.0, exp_mean=1.3592644, exp_median=0.6174331), ), ] @pytest.mark.parametrize("test_case", INTERPOLATION_TEST_CASES) -def test_interpolation(test_case, test_patch): +def test_interpolation(test_case: InterpolationTestCase, test_patch): eopatch = test_case.execute(test_patch) + delta = 1e-4 if isinstance(test_case.task, KrigingInterpolationTask) else 1e-5 # Check types and shapes assert isinstance(eopatch.timestamp, list), "Expected a list of timestamps" @@ -335,14 +286,9 @@ def test_interpolation(test_case, test_patch): assert eopatch.data["NDVI"].shape == (test_case.result_len, 20, 20, 1) # Check results - delta = 1e-5 # Can't be higher accuracy because of Kriging interpolation feature_type, feature_name, _ = test_case.task.renamed_feature data = eopatch[feature_type, feature_name] - - assert np.min(data) == approx(test_case.img_min, abs=delta) - assert np.max(data) == approx(test_case.img_max, abs=delta) - assert np.mean(data) == approx(test_case.img_mean, abs=delta) - assert np.median(data) == approx(test_case.img_median, abs=delta) + test_numpy_data(data, **test_case.expected_statistics, delta=delta) @pytest.mark.parametrize("test_case", COPY_FEATURE_CASES) @@ -359,4 +305,4 @@ def test_copied_fields(test_case, test_patch): (FeatureType.MASK_TIMELESS, "LULC"), ] for feature in copied_features: - assert feature in eopatch.get_feature_list(), f"Expected feature `{feature}` is not present in EOPatch" + assert feature in eopatch, f"Expected feature `{feature}` is not present in EOPatch" diff --git a/features/eolearn/tests/test_local_binary_pattern.py b/features/eolearn/tests/test_local_binary_pattern.py index a8b587893..4915d8ee2 100644 --- a/features/eolearn/tests/test_local_binary_pattern.py +++ b/features/eolearn/tests/test_local_binary_pattern.py @@ -9,34 +9,31 @@ import copy -import numpy as np import pytest -from numpy.testing import assert_array_equal -from pytest import approx + +from sentinelhub.testing_utils import test_numpy_data from eolearn.core import FeatureType -from eolearn.core.eodata_io import FeatureIO from eolearn.features import LocalBinaryPatternTask +LBP_FEATURE = (FeatureType.DATA, "NDVI", "lbp") +OUTPUT_FEATURE = (FeatureType.DATA, "lbp") + @pytest.mark.parametrize( - "task, expected_min, expected_max, expected_mean, expected_median", - ([LocalBinaryPatternTask((FeatureType.DATA, "NDVI", "lbp"), nb_points=24, radius=3), 0.0, 25.0, 15.8313, 21.0],), + "task, expected_statistics", + ( + [ + LocalBinaryPatternTask(LBP_FEATURE, nb_points=24, radius=3), + {"exp_min": 0.0, "exp_max": 25.0, "exp_mean": 15.8313, "exp_median": 21.0}, + ], + ), ) -def test_local_binary_pattern(small_ndvi_eopatch, task, expected_min, expected_max, expected_mean, expected_median): +def test_local_binary_pattern(small_ndvi_eopatch, task, expected_statistics): eopatch = copy.deepcopy(small_ndvi_eopatch) task.execute(eopatch) - # Test that no other features were modified - for feature, value in small_ndvi_eopatch.data.items(): - if isinstance(value, FeatureIO): - value = value.load() - assert_array_equal(value, eopatch.data[feature], err_msg=f"EOPatch data feature '{feature}' has changed") - - delta = 1e-4 + test_numpy_data(eopatch[OUTPUT_FEATURE], exp_shape=(10, 20, 20, 1), **expected_statistics, delta=1e-4) - haralick = eopatch.data["lbp"] - assert np.min(haralick) == approx(expected_min, abs=delta) - assert np.max(haralick) == approx(expected_max, abs=delta) - assert np.mean(haralick) == approx(expected_mean, abs=delta) - assert np.median(haralick) == approx(expected_median, abs=delta) + del eopatch[OUTPUT_FEATURE] + assert small_ndvi_eopatch == eopatch, "Other features of the EOPatch were affected." diff --git a/features/eolearn/tests/test_radiometric_normalization.py b/features/eolearn/tests/test_radiometric_normalization.py index 00047b24b..af2ebbf52 100644 --- a/features/eolearn/tests/test_radiometric_normalization.py +++ b/features/eolearn/tests/test_radiometric_normalization.py @@ -11,11 +11,10 @@ import numpy as np import pytest -from numpy.testing import assert_array_equal -from pytest import approx + +from sentinelhub.testing_utils import test_numpy_data from eolearn.core import FeatureType -from eolearn.core.eodata_io import FeatureIO from eolearn.features import ( BlueCompositingTask, HistogramMatchingTask, @@ -47,7 +46,7 @@ def eopatch_fixture(example_eopatch): @pytest.mark.parametrize( - "task, test_feature, expected_min, expected_max, expected_mean, expected_median", + "task, test_feature, expected_statistics", ( [ MaskFeatureTask( @@ -56,20 +55,14 @@ def eopatch_fixture(example_eopatch): mask_values=[0, 1, 2, 3, 8, 9, 10, 11], ), DATA_TEST_FEATURE, - 0.0002, - 1.4244, - 0.21167801, - 0.142, + {"exp_min": 0.0002, "exp_max": 1.4244, "exp_mean": 0.21167801, "exp_median": 0.1422}, ], [ ReferenceScenesTask( (FeatureType.DATA, "BANDS-S2-L1C", "TEST"), (FeatureType.SCALAR, "CLOUD_COVERAGE"), max_scene_number=5 ), DATA_TEST_FEATURE, - 0.0005, - 0.5318, - 0.16823094, - 0.1404, + {"exp_min": 0.0005, "exp_max": 0.5318, "exp_mean": 0.16823094, "exp_median": 0.1404}, ], [ BlueCompositingTask( @@ -79,10 +72,7 @@ def eopatch_fixture(example_eopatch): interpolation="geoville", ), DATA_TIMELESS_TEST_FEATURE, - 0.0005, - 0.5075, - 0.11658352, - 0.0833, + {"exp_min": 0.0005, "exp_max": 0.5075, "exp_mean": 0.11658352, "exp_median": 0.0833}, ], [ HOTCompositingTask( @@ -93,10 +83,7 @@ def eopatch_fixture(example_eopatch): interpolation="geoville", ), DATA_TIMELESS_TEST_FEATURE, - 0.0005, - 0.5075, - 0.117758796, - 0.0846, + {"exp_min": 0.0005, "exp_max": 0.5075, "exp_mean": 0.117758796, "exp_median": 0.0846}, ], [ MaxNDVICompositingTask( @@ -107,10 +94,7 @@ def eopatch_fixture(example_eopatch): interpolation="geoville", ), DATA_TIMELESS_TEST_FEATURE, - 0.0005, - 0.5075, - 0.13430128, - 0.0941, + {"exp_min": 0.0005, "exp_max": 0.5075, "exp_mean": 0.13430128, "exp_median": 0.0941}, ], [ MaxNDWICompositingTask( @@ -121,10 +105,7 @@ def eopatch_fixture(example_eopatch): interpolation="geoville", ), DATA_TIMELESS_TEST_FEATURE, - 0.0005, - 0.5318, - 0.2580135, - 0.2888, + {"exp_min": 0.0005, "exp_max": 0.5318, "exp_mean": 0.2580135, "exp_median": 0.2888}, ], [ MaxRatioCompositingTask( @@ -136,39 +117,25 @@ def eopatch_fixture(example_eopatch): interpolation="geoville", ), DATA_TIMELESS_TEST_FEATURE, - 0.0006, - 0.5075, - 0.13513365, - 0.0958, + {"exp_min": 0.0006, "exp_max": 0.5075, "exp_mean": 0.13513365, "exp_median": 0.0958}, ], [ HistogramMatchingTask( (FeatureType.DATA, "BANDS-S2-L1C", "TEST"), (FeatureType.DATA_TIMELESS, "REFERENCE_COMPOSITE") ), DATA_TEST_FEATURE, - -0.049050678, - 0.68174845, - 0.1165936, - 0.08370649, + {"exp_min": -0.049050678, "exp_max": 0.68174845, "exp_mean": 0.1165936, "exp_median": 0.08370649}, ], ), ) -def test_haralick(eopatch, task, test_feature, expected_min, expected_max, expected_mean, expected_median): +def test_radiometric_normalization(eopatch, task, test_feature, expected_statistics): initial_patch = copy.deepcopy(eopatch) eopatch = task.execute(eopatch) - # Test that no other features were modified - for feature, value in initial_patch.data.items(): - if isinstance(value, FeatureIO): - value = value.load() - assert_array_equal(value, eopatch.data[feature], err_msg=f"EOPatch data feature '{feature}' has changed") - assert isinstance(eopatch.timestamp, list), "Expected a list of timestamps" assert isinstance(eopatch.timestamp[0], datetime), "Expected timestamps of type datetime.datetime" - delta = 1e-3 - result = eopatch[test_feature] - assert np.nanmin(result) == approx(expected_min, abs=delta) - assert np.nanmax(result) == approx(expected_max, abs=delta) - assert np.nanmean(result) == approx(expected_mean, abs=delta) - assert np.nanmedian(result) == approx(expected_median, abs=delta) + test_numpy_data(eopatch[test_feature], **expected_statistics, delta=1e-3) + + del eopatch[test_feature] + assert initial_patch == eopatch, "Other features of the EOPatch were affected." From 120c2004fb7d1697773b6109da7f258fcccba92c Mon Sep 17 00:00:00 2001 From: jgersak <112631680+jgersak@users.noreply.github.com> Date: Mon, 21 Nov 2022 14:44:20 +0100 Subject: [PATCH 11/14] Adapt test suite to use sh-py testing util for numpy stats (#506) * Adapt test suite to use sh-py testing util for numpy stats * update delta --- geometry/eolearn/tests/test_superpixel.py | 40 ++--- .../eolearn/tests/test_transformations.py | 141 +++++++++--------- 2 files changed, 81 insertions(+), 100 deletions(-) diff --git a/geometry/eolearn/tests/test_superpixel.py b/geometry/eolearn/tests/test_superpixel.py index 5f6af4260..58f5a34cc 100644 --- a/geometry/eolearn/tests/test_superpixel.py +++ b/geometry/eolearn/tests/test_superpixel.py @@ -9,6 +9,8 @@ import numpy as np import pytest +from sentinelhub.testing_utils import test_numpy_data + from eolearn.core import FeatureType from eolearn.geometry import FelzenszwalbSegmentationTask, SlicSegmentationTask, SuperpixelSegmentationTask @@ -16,32 +18,23 @@ @pytest.mark.parametrize( - "task, expected_min, expected_max, expected_mean, expected_median", + "task, expected_statistics", ( [ SuperpixelSegmentationTask( (FeatureType.DATA, "BANDS-S2-L1C"), SUPERPIXEL_FEATURE, scale=100, sigma=0.5, min_size=100 ), - 0, - 25, - 10.6809, - 11, + {"exp_dtype": np.int64, "exp_min": 0, "exp_max": 25, "exp_mean": 10.6809, "exp_median": 11}, ], [ FelzenszwalbSegmentationTask( (FeatureType.DATA_TIMELESS, "MAX_NDVI"), SUPERPIXEL_FEATURE, scale=21, sigma=1.0, min_size=52 ), - 0, - 22, - 8.5302, - 7, + {"exp_dtype": np.int64, "exp_min": 0, "exp_max": 22, "exp_mean": 8.5302, "exp_median": 7}, ], [ FelzenszwalbSegmentationTask((FeatureType.MASK, "CLM"), SUPERPIXEL_FEATURE, scale=1, sigma=0, min_size=15), - 0, - 171, - 86.46267, - 90, + {"exp_dtype": np.int64, "exp_min": 0, "exp_max": 171, "exp_mean": 86.46267, "exp_median": 90}, ], [ SlicSegmentationTask( @@ -52,10 +45,7 @@ max_num_iter=20, sigma=0.8, ), - 0, - 48, - 24.6072, - 25, + {"exp_dtype": np.int64, "exp_min": 0, "exp_max": 48, "exp_mean": 24.6072, "exp_median": 25}, ], [ SlicSegmentationTask( @@ -66,22 +56,12 @@ max_num_iter=7, sigma=0.2, ), - 0, - 195, - 100.1844, - 101, + {"exp_dtype": np.int64, "exp_min": 0, "exp_max": 195, "exp_mean": 100.1844, "exp_median": 101}, ], ), ) -def test_superpixel(test_eopatch, task, expected_min, expected_max, expected_mean, expected_median): +def test_superpixel(test_eopatch, task, expected_statistics): task.execute(test_eopatch) result = test_eopatch[SUPERPIXEL_FEATURE] - assert result.dtype == np.int64, "Expected int64 dtype for result" - - delta = 1e-3 - - assert np.amin(result) == pytest.approx(expected_min, delta), "Minimum values do not match." - assert np.amax(result) == pytest.approx(expected_max, delta), "Maxmum values do not match." - assert np.mean(result) == pytest.approx(expected_mean, delta), "Mean values do not match." - assert np.median(result) == pytest.approx(expected_median, delta), "Median values do not match." + test_numpy_data(result, **expected_statistics, delta=1e-4) diff --git a/geometry/eolearn/tests/test_transformations.py b/geometry/eolearn/tests/test_transformations.py index d151bd0dc..74553b8df 100644 --- a/geometry/eolearn/tests/test_transformations.py +++ b/geometry/eolearn/tests/test_transformations.py @@ -10,7 +10,7 @@ import dataclasses import warnings from functools import partial -from typing import Any, Optional, Type +from typing import Any, Dict, Optional, Tuple, Type, Union import numpy as np import pytest @@ -19,6 +19,8 @@ from numpy.testing import assert_array_equal from shapely.geometry import Polygon +from sentinelhub.testing_utils import test_numpy_data + from eolearn.core import EOPatch, EOTask, FeatureType from eolearn.core.exceptions import EORuntimeWarning from eolearn.geometry import RasterToVectorTask, VectorToRasterTask @@ -40,12 +42,7 @@ class VectorToRasterTestCase: name: str task: EOTask - img_shape: tuple - img_max: int - img_min: int = 0 - img_mean: Optional[float] = None - img_median: Optional[int] = None - img_dtype: type = np.uint8 + img_exp_statistics: Dict[str, Union[Tuple[int, ...], Type, np.dtype, float]] warning: Optional[Type[Warning]] = None @@ -59,10 +56,7 @@ class VectorToRasterTestCase: raster_shape=(FeatureType.DATA, "BANDS-S2-L1C"), no_data_value=20, ), - img_max=8, - img_mean=2.33267, - img_median=2, - img_shape=(101, 100, 1), + img_exp_statistics={"exp_shape": (101, 100, 1), "exp_max": 8, "exp_mean": 2.33267, "exp_median": 2}, ), VectorToRasterTestCase( name="basic test timed", @@ -73,11 +67,13 @@ class VectorToRasterTestCase: raster_shape=(FeatureType.DATA, "BANDS-S2-L1C"), no_data_value=20, ), - img_min=1, - img_max=20, - img_mean=12.4854, - img_median=20, - img_shape=(68, 101, 100, 1), + img_exp_statistics={ + "exp_shape": (68, 101, 100, 1), + "exp_min": 1, + "exp_max": 20, + "exp_mean": 12.4854, + "exp_median": 20, + }, warning=EORuntimeWarning, ), VectorToRasterTestCase( @@ -92,12 +88,14 @@ class VectorToRasterTestCase: write_to_existing=True, raster_dtype=np.int32, ), - img_min=8, - img_max=20, - img_mean=19.76, - img_median=20, - img_dtype=np.int32, - img_shape=(50, 50, 1), + img_exp_statistics={ + "exp_shape": (50, 50, 1), + "exp_dtype": np.int32, + "exp_min": 8, + "exp_max": 20, + "exp_mean": 19.76, + "exp_median": 20, + }, ), VectorToRasterTestCase( name="multiple values filter, resolution, all touched", @@ -112,12 +110,14 @@ class VectorToRasterTestCase: all_touched=True, write_to_existing=False, ), - img_min=1, - img_max=13, - img_mean=12.7093, - img_median=13, - img_dtype=np.uint16, - img_shape=(17, 17, 1), + img_exp_statistics={ + "exp_shape": (17, 17, 1), + "exp_dtype": np.uint16, + "exp_min": 1, + "exp_max": 13, + "exp_mean": 12.7093, + "exp_median": 13, + }, ), VectorToRasterTestCase( name="deprecated parameters, single value, custom resolution", @@ -129,12 +129,14 @@ class VectorToRasterTestCase: no_data_value=-1, raster_dtype=np.int32, ), - img_min=-1, - img_max=14, - img_mean=-0.8411, - img_median=-1, - img_dtype=np.int32, - img_shape=(67, 31, 1), + img_exp_statistics={ + "exp_shape": (67, 31, 1), + "exp_dtype": np.int32, + "exp_min": -1, + "exp_max": 14, + "exp_mean": -0.8411, + "exp_median": -1, + }, ), VectorToRasterTestCase( name="empty vector data test", @@ -145,10 +147,12 @@ class VectorToRasterTestCase: raster_shape=(FeatureType.DATA, "BANDS-S2-L1C"), no_data_value=0, ), - img_max=0, - img_mean=0, - img_median=0, - img_shape=(101, 100, 1), + img_exp_statistics={ + "exp_shape": (101, 100, 1), + "exp_max": 0, + "exp_mean": 0, + "exp_median": 0, + }, ), VectorToRasterTestCase( name="negative polygon buffering", @@ -160,10 +164,12 @@ class VectorToRasterTestCase: raster_shape=(FeatureType.DATA, "BANDS-S2-L1C"), no_data_value=0, ), - img_max=8, - img_mean=0.0229, - img_median=0, - img_shape=(101, 100, 1), + img_exp_statistics={ + "exp_shape": (101, 100, 1), + "exp_max": 8, + "exp_mean": 0.02285, + "exp_median": 0, + }, ), VectorToRasterTestCase( name="positive polygon buffering", @@ -175,10 +181,12 @@ class VectorToRasterTestCase: raster_shape=(FeatureType.DATA, "BANDS-S2-L1C"), no_data_value=0, ), - img_max=8, - img_mean=0.0664, - img_median=0, - img_shape=(101, 100, 1), + img_exp_statistics={ + "exp_shape": (101, 100, 1), + "exp_max": 8, + "exp_mean": 0.0664, + "exp_median": 0, + }, ), VectorToRasterTestCase( name="different crs", @@ -189,10 +197,12 @@ class VectorToRasterTestCase: raster_shape=(FeatureType.DATA, "BANDS-S2-L1C"), no_data_value=0, ), - img_max=8, - img_mean=0.042079, - img_median=0, - img_shape=(101, 100, 1), + img_exp_statistics={ + "exp_shape": (101, 100, 1), + "exp_max": 8, + "exp_mean": 0.042079, + "exp_median": 0, + }, warning=EORuntimeWarning, ), VectorToRasterTestCase( @@ -205,12 +215,14 @@ class VectorToRasterTestCase: no_data_value=-1, raster_dtype=np.dtype("int8"), ), - img_min=-1, - img_max=8, - img_mean=-0.9461386, - img_median=-1, - img_shape=(101, 100, 1), - img_dtype=np.int8, + img_exp_statistics={ + "exp_shape": (101, 100, 1), + "exp_dtype": np.int8, + "exp_min": -1, + "exp_max": 8, + "exp_mean": -0.9461386, + "exp_median": -1, + }, warning=EORuntimeWarning, ), VectorToRasterTestCase( @@ -224,10 +236,7 @@ class VectorToRasterTestCase: no_data_value=0, raster_dtype=bool, ), - img_min=False, - img_max=True, - img_dtype=bool, - img_shape=(100, 150, 1), + img_exp_statistics={"exp_shape": (100, 150, 1), "exp_dtype": bool, "exp_min": False, "exp_max": True}, ), ) @@ -245,16 +254,8 @@ def test_vector_to_raster_result(test_case, test_eopatch): eopatch = test_case.task(test_eopatch) result = eopatch[test_case.task.raster_feature] - delta = 1e-3 - - assert np.amin(result) == pytest.approx(test_case.img_min, abs=delta), "Minimum values do not match." - assert np.amax(result) == pytest.approx(test_case.img_max, abs=delta), "Maximum values do not match." - if test_case.img_mean is not None: - assert np.mean(result) == pytest.approx(test_case.img_mean, abs=delta), "Mean values do not match." - if test_case.img_median is not None: - assert np.median(result) == pytest.approx(test_case.img_median, abs=delta), "Median values do not match." - assert result.dtype == test_case.img_dtype, "Wrong dtype of result." - assert result.shape == test_case.img_shape, "Result is of wrong shape." + + test_numpy_data(result, **test_case.img_exp_statistics, delta=1e-3) def test_polygon_overlap(test_eopatch): From 0ef9850def0d3c9399115d4fe8e7cfdef8c4eaab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Tue, 22 Nov 2022 11:49:26 +0100 Subject: [PATCH 12/14] extend time difference docstring to warn against using large values (#505) --- io/eolearn/io/sentinelhub_process.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/io/eolearn/io/sentinelhub_process.py b/io/eolearn/io/sentinelhub_process.py index 97c08847e..a5da24934 100644 --- a/io/eolearn/io/sentinelhub_process.py +++ b/io/eolearn/io/sentinelhub_process.py @@ -221,7 +221,8 @@ def __init__( :param resolution: Resolution in meters, passed as a single number or a tuple of two numbers - resolution in horizontal and resolution in vertical direction. :param maxcc: Maximum cloud coverage, a float in interval [0, 1] - :param time_difference: Minimum allowed time difference, used when filtering dates, None by default. + :param time_difference: Minimum allowed time difference, used when filtering dates. Also used by the service + for mosaicking, timestamps might be misleading for large values. :param cache_folder: Path to cache_folder. If set to None (default) requests will not be cached. :param config: An instance of SHConfig defining the service :param max_threads: Maximum threads to be used when downloading data. @@ -413,7 +414,8 @@ def __init__( :param additional_data: A list of additional data to be downloaded, such as SCL, SNW, dataMask, etc. :param evalscript: An optional parameter to override an evalscript that is generated by default :param maxcc: Maximum cloud coverage. - :param time_difference: Minimum allowed time difference, used when filtering dates, None by default. + :param time_difference: Minimum allowed time difference, used when filtering dates. Also used by the service + for mosaicking, timestamps might be misleading for large values. :param cache_folder: Path to cache_folder. If set to None (default) requests will not be cached. :param config: An instance of SHConfig defining the service :param max_threads: Maximum threads to be used when downloading data. From 87f930ac4d83f389f0cab51db980e8737ccd2c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Tue, 22 Nov 2022 15:01:58 +0100 Subject: [PATCH 13/14] remove code that tries to automate byoc bands (#507) --- io/eolearn/io/sentinelhub_process.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/io/eolearn/io/sentinelhub_process.py b/io/eolearn/io/sentinelhub_process.py index a5da24934..32d8f4fab 100644 --- a/io/eolearn/io/sentinelhub_process.py +++ b/io/eolearn/io/sentinelhub_process.py @@ -17,7 +17,6 @@ import numpy as np from sentinelhub import ( - Band, BBox, DataCollection, Geometry, @@ -29,7 +28,6 @@ SentinelHubRequest, SentinelHubSession, SHConfig, - Unit, bbox_to_dimensions, filter_times, parse_time_interval, @@ -79,7 +77,7 @@ def __init__( self.resolution = resolution self.config = config or SHConfig() self.max_threads = max_threads - self.data_collection = DataCollection(data_collection) + self.data_collection: DataCollection = DataCollection(data_collection) self.cache_folder = cache_folder self.session_loader = session_loader self.upsampling = upsampling @@ -476,8 +474,6 @@ def _parse_requested_bands(self, bands, available_bands): for band_name in bands: if band_name in band_info_dict: requested_bands.append(band_info_dict[band_name]) - elif self.data_collection.is_batch or self.data_collection.is_byoc: - requested_bands.append(Band(band_name, (Unit.DN,), (np.float32,))) else: raise ValueError( f"Data collection {self.data_collection} does not have specifications for {band_name}." From e8f228c1923e35804f298721ec95b90c674ced36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Wed, 23 Nov 2022 15:03:19 +0100 Subject: [PATCH 14/14] increase version to 1.3.1 (#508) --- CHANGELOG.md | 10 ++++++++++ core/eolearn/core/__init__.py | 2 +- .../eolearn/coregistration/__init__.py | 2 +- features/eolearn/features/__init__.py | 2 +- geometry/eolearn/geometry/__init__.py | 2 +- io/eolearn/io/__init__.py | 2 +- mask/eolearn/mask/__init__.py | 2 +- ml_tools/eolearn/ml_tools/__init__.py | 2 +- setup.py | 18 +++++++++--------- .../eolearn/visualization/__init__.py | 2 +- 10 files changed, 27 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a718ebd..728068796 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## [Version 1.3.1] - 2022-11-23 + +- Sentinel Hub IO tasks now support a custom timestamp filtration via `timestamp_filter` parameter. +- `MergeFeatureTask` now supports the `axis` parameter. +- Fix minor issues with the coregistration module. +- Prepare for future removal of `sentinelhub.os_utils`. +- Fix type annotations after `mypy` update. +- Improvements to tests and various minor changes. + + ## [Version 1.3.0] - 2022-10-06 - (**codebreaking**) Adapted Sentinel Hub tasks to `sentinelhub-py 3.8.0` which switched to Catalog 1.0.0. diff --git a/core/eolearn/core/__init__.py b/core/eolearn/core/__init__.py index f9347af56..535f84de7 100644 --- a/core/eolearn/core/__init__.py +++ b/core/eolearn/core/__init__.py @@ -32,4 +32,4 @@ from .utils.parallelize import execute_with_mp_lock, join_futures, join_futures_iter, parallelize from .utils.parsing import FeatureParser -__version__ = "1.3.0" +__version__ = "1.3.1" diff --git a/coregistration/eolearn/coregistration/__init__.py b/coregistration/eolearn/coregistration/__init__.py index a7dad3f22..98b42917f 100644 --- a/coregistration/eolearn/coregistration/__init__.py +++ b/coregistration/eolearn/coregistration/__init__.py @@ -4,4 +4,4 @@ from .coregistration import ECCRegistrationTask, InterpolationType, PointBasedRegistrationTask, RegistrationTask -__version__ = "1.3.0" +__version__ = "1.3.1" diff --git a/features/eolearn/features/__init__.py b/features/eolearn/features/__init__.py index 5b0567c76..49d350f32 100644 --- a/features/eolearn/features/__init__.py +++ b/features/eolearn/features/__init__.py @@ -38,4 +38,4 @@ AddSpatioTemporalFeaturesTask, ) -__version__ = "1.3.0" +__version__ = "1.3.1" diff --git a/geometry/eolearn/geometry/__init__.py b/geometry/eolearn/geometry/__init__.py index 66f733ee6..928e17459 100644 --- a/geometry/eolearn/geometry/__init__.py +++ b/geometry/eolearn/geometry/__init__.py @@ -11,4 +11,4 @@ ) from .transformations import RasterToVectorTask, VectorToRasterTask -__version__ = "1.3.0" +__version__ = "1.3.1" diff --git a/io/eolearn/io/__init__.py b/io/eolearn/io/__init__.py index 589592fc9..96623c592 100644 --- a/io/eolearn/io/__init__.py +++ b/io/eolearn/io/__init__.py @@ -13,4 +13,4 @@ get_available_timestamps, ) -__version__ = "1.3.0" +__version__ = "1.3.1" diff --git a/mask/eolearn/mask/__init__.py b/mask/eolearn/mask/__init__.py index fcba92689..bda30a8f7 100644 --- a/mask/eolearn/mask/__init__.py +++ b/mask/eolearn/mask/__init__.py @@ -8,4 +8,4 @@ from .snow_mask import SnowMaskTask, TheiaSnowMaskTask from .utils import resize_images -__version__ = "1.3.0" +__version__ = "1.3.1" diff --git a/ml_tools/eolearn/ml_tools/__init__.py b/ml_tools/eolearn/ml_tools/__init__.py index f7a309be5..2e5366540 100644 --- a/ml_tools/eolearn/ml_tools/__init__.py +++ b/ml_tools/eolearn/ml_tools/__init__.py @@ -5,4 +5,4 @@ from .sampling import BlockSamplingTask, FractionSamplingTask, GridSamplingTask, sample_by_values from .train_test_split import TrainTestSplitTask -__version__ = "1.3.0" +__version__ = "1.3.1" diff --git a/setup.py b/setup.py index 60036dc2f..04a3360b5 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def parse_requirements(file): setup( name="eo-learn", python_requires=">=3.7", - version="1.3.0", + version="1.3.1", description="Earth observation processing framework for machine learning in Python", long_description=get_long_description(), long_description_content_type="text/markdown", @@ -38,14 +38,14 @@ def parse_requirements(file): packages=[], include_package_data=True, install_requires=[ - "eo-learn-core==1.3.0", - "eo-learn-coregistration==1.3.0", - "eo-learn-features==1.3.0", - "eo-learn-geometry==1.3.0", - "eo-learn-io==1.3.0", - "eo-learn-mask==1.3.0", - "eo-learn-ml-tools==1.3.0", - "eo-learn-visualization==1.3.0", + "eo-learn-core==1.3.1", + "eo-learn-coregistration==1.3.1", + "eo-learn-features==1.3.1", + "eo-learn-geometry==1.3.1", + "eo-learn-io==1.3.1", + "eo-learn-mask==1.3.1", + "eo-learn-ml-tools==1.3.1", + "eo-learn-visualization==1.3.1", ], extras_require={"DEV": parse_requirements("requirements-dev.txt")}, zip_safe=False, diff --git a/visualization/eolearn/visualization/__init__.py b/visualization/eolearn/visualization/__init__.py index d337c77d9..3bb090733 100644 --- a/visualization/eolearn/visualization/__init__.py +++ b/visualization/eolearn/visualization/__init__.py @@ -4,4 +4,4 @@ from .eopatch import PlotBackend, PlotConfig -__version__ = "1.3.0" +__version__ = "1.3.1"