From 52fd3fcfa88ef479307ec1d7a138ff22298cbfa8 Mon Sep 17 00:00:00 2001 From: iovsn Date: Sun, 16 Feb 2020 17:31:03 +0100 Subject: [PATCH 1/8] use the new SentinelHubRequest for Processing API --- io/eolearn/io/processing_api.py | 94 +++++++++++++++------------------ 1 file changed, 42 insertions(+), 52 deletions(-) diff --git a/io/eolearn/io/processing_api.py b/io/eolearn/io/processing_api.py index cf5c99350..54e7efd73 100644 --- a/io/eolearn/io/processing_api.py +++ b/io/eolearn/io/processing_api.py @@ -6,9 +6,8 @@ import datetime as dt import numpy as np -from sentinelhub import WebFeatureService, MimeType, SentinelHubDownloadClient, DownloadRequest, SHConfig,\ +from sentinelhub import SentinelHubRequest, WebFeatureService, MimeType, SentinelHubDownloadClient, SHConfig, \ bbox_to_dimensions, parse_time_interval, DataSource -import sentinelhub.sentinelhub_request as shr from sentinelhub.time_utils import iso_to_datetime from eolearn.core import EOPatch, EOTask, FeatureType @@ -43,15 +42,7 @@ def __init__(self, data_source, size=None, resolution=None, cache_folder=None, c self.config = config or SHConfig() self.max_threads = max_threads self.data_source = data_source - - self.request_args = dict( - url=self.config.get_sh_processing_api_url(), - headers={"accept": "application/tar", 'content-type': 'application/json'}, - data_folder=cache_folder, - hash_save=bool(cache_folder), - request_type='POST', - data_type=MimeType.TAR - ) + self.cache_folder = cache_folder def execute(self, eopatch=None, bbox=None, time_interval=None): """ Main execute method for the Processing API tasks @@ -80,8 +71,8 @@ def execute(self, eopatch=None, bbox=None, time_interval=None): elif timestamp: eopatch.timestamp = timestamp - payloads = self._build_payloads(bbox, size_x, size_y, timestamp, time_interval) - requests = [DownloadRequest(post_values=payload, **self.request_args) for payload in payloads] + requests = self._build_requests(bbox, size_x, size_y, timestamp, time_interval) + requests = [request._download_request for request in requests] LOGGER.debug('Downloading %d requests of type %s', len(requests), str(self.data_source)) client = SentinelHubDownloadClient(config=self.config) @@ -118,10 +109,10 @@ def _extract_data(self, eopatch, images, shape): """ raise NotImplementedError("The _extract_data method should be implemented by the subclass.") - def _build_payloads(self, bbox, size_x, size_y, timestamp, time_interval): - """ Build payloads for the requests to the service + def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval): + """ Build requests """ - raise NotImplementedError("The _build_payloads method should be implemented by the subclass.") + raise NotImplementedError("The _build_requests method should be implemented by the subclass.") def _get_timestamp(self, time_interval, bbox): """ Get the timestamp array needed as a parameter for downloading the images @@ -318,39 +309,37 @@ def _get_timestamp(self, time_interval, bbox): return [dates[0]] + [d2 for d1, d2 in zip(dates[:-1], dates[1:]) if d2 - d1 > self.time_difference] - def _build_payloads(self, bbox, size_x, size_y, timestamp, time_interval): - """ Build payloads for the requests to the service + def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval): + """ Build requests """ if self.single_scene: dates = [(iso_to_datetime(time_interval[0]), iso_to_datetime(time_interval[1]))] else: dates = [(date - self.time_difference, date + self.time_difference) for date in timestamp] - return [self._request_payload(date1, date2, bbox, size_x, size_y) for date1, date2 in dates] + return [self._create_sh_request(date1, date2, bbox, size_x, size_y) for date1, date2 in dates] - def _request_payload(self, date_from, date_to, bbox, size_x, size_y): - """ Build the payload dictionary for the request + def _create_sh_request(self, date_from, date_to, bbox, size_x, size_y): + """ Create an instance of SentinelHubRequest """ - time_from, time_to = date_from.isoformat() + 'Z', date_to.isoformat() + 'Z' - - responses = [shr.response(btype.id, 'image/tiff') for btype in self.requested_bands] - - responses.append(shr.response('userdata', 'application/json')) - - data_type = 'CUSTOM' if self.data_source.is_custom() else self.data_source.api_identifier() - - data = shr.data(time_from=time_from, time_to=time_to, data_type=data_type) - data['dataFilter']['maxCloudCoverage'] = int(self.maxcc * 100) - data['dataFilter']['mosaickingOrder'] = self.mosaicking_order - - if data_type == 'CUSTOM': - data['dataFilter']['collectionId'] = self.data_source.value - - return shr.body( - request_bounds=shr.bounds(crs=bbox.crs.opengis_string, bbox=list(bbox)), - request_data=[data], - request_output=shr.output(size_x=size_x, size_y=size_y, responses=responses), - evalscript=self.generate_evalscript() + responses = [SentinelHubRequest.output_response(btype.id, 'image/tiff') for btype in self.requested_bands] + responses.append(SentinelHubRequest.output_response('userdata', 'application/json')) + + return SentinelHubRequest( + evalscript=self.generate_evalscript(), + input_data=[ + SentinelHubRequest.input_data( + data_source=self.data_source, + time_interval=(date_from, date_to), + mosaicking_order=self.mosaicking_order, + maxcc=self.maxcc + ) + ], + responses=responses, + bounds=bbox, + size=(size_x, size_y), + mime_type=MimeType.TAR, + data_folder=self.cache_folder ) def _extract_data(self, eopatch, images, shape): @@ -440,8 +429,8 @@ def __init__(self, dem_feature, size=None, resolution=None, cache_folder=None, c self.dem_feature = next(feature_parser()) - def _build_payloads(self, bbox, size_x, size_y, timestamp, time_interval): - """ Build payloads for the requests to the service + def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval): + """ Build requests """ evalscript = """ //VERSION=3 @@ -462,19 +451,20 @@ def _build_payloads(self, bbox, size_x, size_y, timestamp, time_interval): } """ - responses = [shr.response('default', 'image/tiff'), shr.response('userdata', 'application/json')] - request_body = shr.body( - request_bounds=shr.bounds(crs=bbox.crs.opengis_string, bbox=list(bbox)), - request_data=[{"type": "DEM"}], - request_output=shr.output(size_x=size_x, size_y=size_y, responses=responses), - evalscript=evalscript + request = SentinelHubRequest( + evalscript=evalscript, + input_data=[SentinelHubRequest.input_data(data_source=self.data_source)], + responses=[SentinelHubRequest.output_response('default', 'image/tiff')], + bounds=bbox, + size=(size_x, size_y), + mime_type=MimeType.TIFF, + data_folder=self.cache_folder ) - return [request_body] + return [request] def _extract_data(self, eopatch, images, shape): """ Extract data from the received images and assign them to eopatch features """ - tif = images[0]['default.tif'] - + tif = images[0] eopatch[self.dem_feature] = tif[..., np.newaxis].astype(np.int16) From 247c701cd52a8bad2ebe6a738c232f5147a8fe92 Mon Sep 17 00:00:00 2001 From: iovsn Date: Mon, 17 Feb 2020 14:14:16 +0100 Subject: [PATCH 2/8] catch up with the proc api changes in sh-py --- io/eolearn/io/processing_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/io/eolearn/io/processing_api.py b/io/eolearn/io/processing_api.py index 54e7efd73..8bc311670 100644 --- a/io/eolearn/io/processing_api.py +++ b/io/eolearn/io/processing_api.py @@ -72,7 +72,7 @@ def execute(self, eopatch=None, bbox=None, time_interval=None): eopatch.timestamp = timestamp requests = self._build_requests(bbox, size_x, size_y, timestamp, time_interval) - requests = [request._download_request for request in requests] + requests = [request.download_list[0] for request in requests] LOGGER.debug('Downloading %d requests of type %s', len(requests), str(self.data_source)) client = SentinelHubDownloadClient(config=self.config) @@ -336,7 +336,7 @@ def _create_sh_request(self, date_from, date_to, bbox, size_x, size_y): ) ], responses=responses, - bounds=bbox, + bbox=bbox, size=(size_x, size_y), mime_type=MimeType.TAR, data_folder=self.cache_folder @@ -455,7 +455,7 @@ def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval): evalscript=evalscript, input_data=[SentinelHubRequest.input_data(data_source=self.data_source)], responses=[SentinelHubRequest.output_response('default', 'image/tiff')], - bounds=bbox, + bbox=bbox, size=(size_x, size_y), mime_type=MimeType.TIFF, data_folder=self.cache_folder From 0af8ae7c315f260284f7518150a9c71209db5980 Mon Sep 17 00:00:00 2001 From: iovsn Date: Mon, 17 Feb 2020 15:03:16 +0100 Subject: [PATCH 3/8] Updated according to sh-py mime_type changes --- io/eolearn/io/processing_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/io/eolearn/io/processing_api.py b/io/eolearn/io/processing_api.py index 8bc311670..2b40f99e7 100644 --- a/io/eolearn/io/processing_api.py +++ b/io/eolearn/io/processing_api.py @@ -322,8 +322,8 @@ def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval): def _create_sh_request(self, date_from, date_to, bbox, size_x, size_y): """ Create an instance of SentinelHubRequest """ - responses = [SentinelHubRequest.output_response(btype.id, 'image/tiff') for btype in self.requested_bands] - responses.append(SentinelHubRequest.output_response('userdata', 'application/json')) + responses = [SentinelHubRequest.output_response(btype.id, MimeType.TIFF) for btype in self.requested_bands] + responses.append(SentinelHubRequest.output_response('userdata', MimeType.JSON)) return SentinelHubRequest( evalscript=self.generate_evalscript(), @@ -454,7 +454,7 @@ def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval): request = SentinelHubRequest( evalscript=evalscript, input_data=[SentinelHubRequest.input_data(data_source=self.data_source)], - responses=[SentinelHubRequest.output_response('default', 'image/tiff')], + responses=[SentinelHubRequest.output_response('default', MimeType.TIFF)], bbox=bbox, size=(size_x, size_y), mime_type=MimeType.TIFF, From 94cfc82f7c487c758462eea1fd4af1abc1674903 Mon Sep 17 00:00:00 2001 From: iovsn Date: Mon, 17 Feb 2020 17:03:59 +0100 Subject: [PATCH 4/8] removed the mime_type SentinelHubRequest parameter --- io/eolearn/io/processing_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/io/eolearn/io/processing_api.py b/io/eolearn/io/processing_api.py index 2b40f99e7..1439057dd 100644 --- a/io/eolearn/io/processing_api.py +++ b/io/eolearn/io/processing_api.py @@ -338,7 +338,6 @@ def _create_sh_request(self, date_from, date_to, bbox, size_x, size_y): responses=responses, bbox=bbox, size=(size_x, size_y), - mime_type=MimeType.TAR, data_folder=self.cache_folder ) @@ -457,7 +456,6 @@ def _build_requests(self, bbox, size_x, size_y, timestamp, time_interval): responses=[SentinelHubRequest.output_response('default', MimeType.TIFF)], bbox=bbox, size=(size_x, size_y), - mime_type=MimeType.TIFF, data_folder=self.cache_folder ) From 4d439e40acf3191c8396c154df551dcf3a62f645 Mon Sep 17 00:00:00 2001 From: iovsn Date: Mon, 17 Feb 2020 17:33:29 +0100 Subject: [PATCH 5/8] Raised version to 0.7.2 --- CHANGELOG.md | 6 +++++- core/eolearn/core/__init__.py | 2 +- io/eolearn/io/__init__.py | 2 +- io/eolearn/io/geopedia.py | 6 +++--- io/setup.py | 2 +- setup.py | 9 ++++----- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0511dd7ec..f03629ec5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ -## [Unreleased] +## [Version 0.7.2] - 2019-02-17 +- Support additional data in the Processing API input task (such as sunAzimuthAngles, sunZenithAngles, viewAzimuthMean, viewZenithMean) +- Compatibility with the `sentinelhub-py` 3.0 +- Removed support for python 3.5 +- Multiprocessing Log filtering ## [Version 0.7.1] - 2019-02-05 ### Fixed diff --git a/core/eolearn/core/__init__.py b/core/eolearn/core/__init__.py index c04dff750..256330846 100644 --- a/core/eolearn/core/__init__.py +++ b/core/eolearn/core/__init__.py @@ -18,4 +18,4 @@ from .utilities import deep_eq, negate_mask, constant_pad, get_common_timestamps, bgr_to_rgb, FeatureParser -__version__ = '0.7.1' +__version__ = '0.7.2' diff --git a/io/eolearn/io/__init__.py b/io/eolearn/io/__init__.py index ec77f2724..6c3f18ea4 100644 --- a/io/eolearn/io/__init__.py +++ b/io/eolearn/io/__init__.py @@ -9,4 +9,4 @@ from .local_io import ExportToTiff, ImportFromTiff from .processing_api import SentinelHubInputTask, SentinelHubDemTask -__version__ = '0.7.1' +__version__ = '0.7.2' diff --git a/io/eolearn/io/geopedia.py b/io/eolearn/io/geopedia.py index 7094d437e..254f4807a 100644 --- a/io/eolearn/io/geopedia.py +++ b/io/eolearn/io/geopedia.py @@ -16,7 +16,7 @@ import rasterio.transform import rasterio.warp -from sentinelhub import MimeType, CustomUrlParam, CRS, GeopediaWmsRequest, transform_bbox +from sentinelhub import MimeType, CustomUrlParam, CRS, GeopediaWmsRequest from eolearn.core import EOTask, FeatureType @@ -55,7 +55,7 @@ def _get_wms_request(self, bbox, size_x, size_y): """ Returns WMS request. """ - bbox_3857 = transform_bbox(bbox, CRS.POP_WEB) + bbox_3857 = bbox.transform(CRS.POP_WEB) return GeopediaWmsRequest(layer=self.layer, theme=self.theme, @@ -79,7 +79,7 @@ def _reproject(self, eopatch, src_raster): dst_raster = np.ones((height, width), dtype=self.raster_dtype) - src_bbox = transform_bbox(eopatch.bbox, CRS.POP_WEB) + src_bbox = eopatch.bbox.transform(CRS.POP_WEB) src_transform = rasterio.transform.from_bounds(*src_bbox, width=width, height=height) dst_bbox = eopatch.bbox diff --git a/io/setup.py b/io/setup.py index 961bd0fba..1299710a8 100644 --- a/io/setup.py +++ b/io/setup.py @@ -28,7 +28,7 @@ def get_version(): setup( name='eo-learn-io', - python_requires='>=3.5', + python_requires='>=3.6', version=get_version(), description='A collection of input/output utilities and EOTasks', long_description=get_long_description(), diff --git a/setup.py b/setup.py index c1684690b..5567b7de4 100644 --- a/setup.py +++ b/setup.py @@ -21,8 +21,8 @@ def parse_requirements(file): setup( name='eo-learn', - python_requires='>=3.5', - version='0.7.1', + python_requires='>=3.6', + version='0.7.2', description='Earth observation processing framework for machine learning in Python', long_description=get_long_description(), long_description_content_type='text/markdown', @@ -33,11 +33,11 @@ def parse_requirements(file): packages=[], include_package_data=True, install_requires=[ - 'eo-learn-core>=0.7.1', + 'eo-learn-core>=0.7.2', 'eo-learn-coregistration>=0.7.0', 'eo-learn-features>=0.7.0', 'eo-learn-geometry>=0.7.0', - 'eo-learn-io>=0.7.1', + 'eo-learn-io>=0.7.2', 'eo-learn-mask>=0.7.0', 'eo-learn-ml-tools>=0.7.0', 'eo-learn-visualization>=0.7.1' @@ -57,7 +57,6 @@ def parse_requirements(file): 'Operating System :: Unix', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', From f3bd84e0edbc855b00528fb1c5f0cb1767e5cf32 Mon Sep 17 00:00:00 2001 From: AleksMat Date: Mon, 17 Feb 2020 18:14:09 +0100 Subject: [PATCH 6/8] a few changes in Travis CI --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c87ee033e..a25db045c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,14 +5,11 @@ notifications: email: false python: -- "3.5" - "3.6" - "3.7" - "3.8" before_install: -- sudo add-apt-repository -y ppa:jonathonf/backports -- sudo apt-get -qq update - sudo apt-get install -y libgdal-dev - sudo apt-get install graphviz - sudo apt-get install proj-bin @@ -26,6 +23,7 @@ install: else python install_all.py; fi +- pip install geopandas==0.6.3 # This is temporal fix script: - sentinelhub.config --instance_id $INSTANCE_ID From e0b1ffc80a885f3a5d90fbd5752e77acfd152aab Mon Sep 17 00:00:00 2001 From: AleksMat Date: Mon, 17 Feb 2020 18:21:10 +0100 Subject: [PATCH 7/8] updated core requirements - sentinelhub version --- core/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/requirements.txt b/core/requirements.txt index f0d63d41c..90a0c1f94 100644 --- a/core/requirements.txt +++ b/core/requirements.txt @@ -2,7 +2,7 @@ attrs>=18.2.0 python-dateutil numpy>=1.16 geopandas -sentinelhub==3.0.0b1 +sentinelhub>=3.0.0 tqdm>=4.27 boto3 fs From 0a4d95ab38459565dbbaab7de5b1b2911584192b Mon Sep 17 00:00:00 2001 From: AleksMat Date: Mon, 17 Feb 2020 18:34:34 +0100 Subject: [PATCH 8/8] updated geometry module with new sentinelhub-py version --- geometry/eolearn/geometry/__init__.py | 2 +- geometry/eolearn/geometry/transformations.py | 2 +- geometry/setup.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/geometry/eolearn/geometry/__init__.py b/geometry/eolearn/geometry/__init__.py index 8cd9e7af4..d8ac98849 100644 --- a/geometry/eolearn/geometry/__init__.py +++ b/geometry/eolearn/geometry/__init__.py @@ -7,4 +7,4 @@ from .superpixel import SuperpixelSegmentation, FelzenszwalbSegmentation, SlicSegmentation, MarkSegmentationBoundaries from .transformations import VectorToRaster, RasterToVector -__version__ = '0.7.0' +__version__ = '0.7.2' diff --git a/geometry/eolearn/geometry/transformations.py b/geometry/eolearn/geometry/transformations.py index e809fd42f..00b685066 100644 --- a/geometry/eolearn/geometry/transformations.py +++ b/geometry/eolearn/geometry/transformations.py @@ -384,7 +384,7 @@ def execute(self, eopatch): affine_transform = rasterio.transform.from_bounds(*eopatch.bbox, width=width, height=height) - crs = eopatch.bbox.get_crs() + crs = eopatch.bbox.crs if raster_ft.is_timeless(): eopatch[vector_ft][vector_fn] = self._vectorize_single_raster(raster, affine_transform, crs) diff --git a/geometry/setup.py b/geometry/setup.py index 6de87cb28..23b68a159 100644 --- a/geometry/setup.py +++ b/geometry/setup.py @@ -28,7 +28,7 @@ def get_version(): setup( name='eo-learn-geometry', - python_requires='>=3.5', + python_requires='>=3.6', version=get_version(), description='A collection of geometry utilities and EOTasks', long_description=get_long_description(), diff --git a/setup.py b/setup.py index 5567b7de4..90e5dac90 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ def parse_requirements(file): 'eo-learn-core>=0.7.2', 'eo-learn-coregistration>=0.7.0', 'eo-learn-features>=0.7.0', - 'eo-learn-geometry>=0.7.0', + 'eo-learn-geometry>=0.7.2', 'eo-learn-io>=0.7.2', 'eo-learn-mask>=0.7.0', 'eo-learn-ml-tools>=0.7.0',