From 4da571b26b9434c42413effd97d7befa2a3e85ee Mon Sep 17 00:00:00 2001 From: AleksMat Date: Fri, 23 Mar 2018 18:48:43 +0100 Subject: [PATCH] version 1.1.0 - added support for all datasources and other minor improvements --- README.md | 2 +- SentinelHub/SentinelHub.py | 189 ++++++++++++++++----- SentinelHub/SentinelHub_dockwidget_base.ui | 6 +- SentinelHub/Settings.py | 47 ++++- SentinelHub/metadata.txt | 10 +- 5 files changed, 189 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index d3ded02..c91c23a 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The plugin currently supports QGIS versions 2.* and 3.0. SentinelHub QGIS Plugin is available in QGIS Official Plugin Repository. For install just open QGIS, select `Plugins -> Manage and Install Plugins` and search for the plugin. -In case of manual installation you can download [latest release](https://github.com/sinergise/qgis_sentinel_hub/releases/latest), unzip it into QGIS Plugin directory (`%userprofile%/.qgis/python/plugins/` for Windows users, `~/.qgis2/python/plugins/` for Linux and OS X users) and enable plugin under QGIS Installed Plugins. +In case of manual installation you can download [latest release](https://github.com/sinergise/qgis_sentinel_hub/releases/latest), unzip it into QGIS Plugin directory and enable plugin under QGIS Installed Plugins. ## Development diff --git a/SentinelHub/SentinelHub.py b/SentinelHub/SentinelHub.py index 280c455..d4d2134 100644 --- a/SentinelHub/SentinelHub.py +++ b/SentinelHub/SentinelHub.py @@ -68,6 +68,10 @@ def is_qgis_version_3(): SUCCESS_MSG = ('Success', Qgis.Success if is_qgis_version_3() else QgsMessageBar.SUCCESS) +class InvalidInstanceId(ValueError): + pass + + class SentinelHub: def __init__(self, iface): @@ -78,6 +82,7 @@ def __init__(self, iface): # initialize plugin directory self.plugin_dir = os.path.dirname(__file__) + self.plugin_version = self.get_plugin_version() # initialize locale locale = QSettings().value('locale/userLocale')[0:2] @@ -100,6 +105,8 @@ def __init__(self, iface): self.toolbar.setObjectName(u'SentinelHub') self.pluginIsActive = False self.dockwidget = None + self.base_url = None + self.data_source = None # Set value self.instance_id = QSettings().value(Settings.instance_id_location) @@ -119,6 +126,7 @@ def __init__(self, iface): for name in ['latMin', 'latMax', 'lngMin', 'lngMax']: self.custom_bbox_params[name] = '' + @staticmethod def translate(message): """Get the translation for a string using Qt translation API. @@ -174,7 +182,7 @@ def init_gui_settings(self): self.set_values() self.dockwidget.priority.clear() - self.dockwidget.priority.addItems(Settings.priority_list) + self.dockwidget.priority.addItems(sorted(Settings.priority_map.keys(), reverse=True)) self.dockwidget.format.clear() self.dockwidget.format.addItems(Settings.img_formats) @@ -190,6 +198,14 @@ def set_values(self): self.dockwidget.lngMin.setText(self.custom_bbox_params['lngMin']) self.dockwidget.lngMax.setText(self.custom_bbox_params['lngMax']) + def get_plugin_version(self): + try: + for line in open(os.path.join(self.plugin_dir, 'metadata.txt')): + if line.startswith('version'): + return line.split("=")[1].strip() + except IOError: + return '?' + # -------------------------------------------------------------------------- def show_message(self, message, message_type): @@ -266,7 +282,7 @@ def get_wms_uri(self): # Every parameter that QGIS layer doesn't use by default must be in url # And url has to be encoded - url = '{}wms/{}?TIME={}&priority={}&maxcc={}'.format(Settings.url_base, self.instance_id, self.get_time(), + url = '{}wms/{}?TIME={}&priority={}&maxcc={}'.format(self.base_url, self.instance_id, self.get_time(), Settings.parameters['priority'], Settings.parameters['maxcc']) return '{}url={}'.format(uri, quote_plus(url)) @@ -279,7 +295,7 @@ def get_wcs_url(self, bbox, crs=None): :param crs: CRS of bounding box :type crs: str or None """ - url = '{}wcs/{}?'.format(Settings.url_base, self.instance_id) + url = '{}wcs/{}?'.format(self.base_url, self.instance_id) request_parameters = list(Settings.parameters_wcs.items()) + list(Settings.parameters.items()) for parameter, value in request_parameters: @@ -293,13 +309,22 @@ def get_wcs_url(self, bbox, crs=None): def get_wfs_url(self, time_range): """ Generate URL for WFS request from parameters """ - url = '{}wfs/{}?'.format(Settings.url_base, self.instance_id) + url = '{}wfs/{}?'.format(self.base_url, self.instance_id) for parameter, value in Settings.parameters_wfs.items(): url += '{}={}&'.format(parameter, value) return '{}bbox={}&time={}&srsname={}&maxcc=100'.format(url, self.bbox_to_string(self.get_bbox()), time_range, Settings.parameters['crs']) + @staticmethod + def get_capabilities_url(base_url, service, instance_id, get_json=False): + """ Generates url for obtaining service capabilities + """ + url = '{}{}/{}?service={}&request=GetCapabilities&version=1.1.1'.format(base_url, service, instance_id, service) + if get_json: + return url + '&format=application/json' + return url + # --------------------------------------------------------------------------- def get_capabilities(self, service, instance_id): @@ -315,38 +340,59 @@ def get_capabilities(self, service, instance_id): if not instance_id: return [], False - response = self.download_from_url('{}{}/{}?service={}&request=GetCapabilities' - '&version=1.1.1'.format(Settings.url_base, service, instance_id, service)) + try: + response = self.download_from_url(self.get_capabilities_url(Settings.services_base_url, service, + instance_id), raise_invalid_id=True) + self.base_url = Settings.services_base_url + except InvalidInstanceId: + response = self.download_from_url(self.get_capabilities_url(Settings.ipt_base_url, service, instance_id)) + self.base_url = Settings.ipt_base_url + props = [] if response: root = ElementTree.fromstring(response.content) for layer in root.findall('./Capability/Layer/Layer'): props.append({'Title': layer.find('Title').text, 'Name': layer.find('Name').text}) + + if self.base_url == Settings.services_base_url: + json_response = self.download_from_url(self.get_capabilities_url(self.base_url, service, instance_id, + get_json=True), raise_invalid_id=True) + try: + layers = json_response.json()['layers'] + for prop, layer in zip(props, layers): + if prop['Name'] == layer['id']: + prop['Dataset'] = layer['dataset'] + # prop['Description'] = layer['description'] + except (ValueError, KeyError): + pass + return props, response is not None - def get_cloud_cover(self, time_range): + def get_cloud_cover(self): """ Get cloud cover for current extent. - - :return: """ self.cloud_cover = {} + self.clear_calendar_cells() if not self.instance_id or len(self.qgis_layers) == 0: return + if self.base_url != Settings.services_base_url: # Uswest is too slow for this + return # Check if area is too large width, height = self.get_bbox_size(self.get_bbox()) if max(width, height) > Settings.max_cloud_cover_image_size: return - response = self.download_from_url(self.get_wfs_url(time_range)) + time_range = self.get_calendar_month_interval() + response = self.download_from_url(self.get_wfs_url(time_range), ignore_exception=True) if response: area_info = response.json() for feature in area_info['features']: - self.cloud_cover.update( - {str(feature['properties']['date']): feature['properties']['cloudCoverPercentage']}) + self.cloud_cover[str(feature['properties']['date'])] = feature['properties'].get('cloudCoverPercentage', + 0) self.update_calendar_from_cloud_cover() # ---------------------------------------------------------------------------- @@ -379,41 +425,64 @@ def download_wcs_data(self, url, filename): else: self.show_message("Failed to download from {} to {}".format(url, filename), CRITICAL_MSG) - def download_from_url(self, url, stream=False): + def download_from_url(self, url, stream=False, raise_invalid_id=False, ignore_exception=False): """ Downloads data from url and handles possible errors :param url: download url :type url: str :param stream: True if download should be streamed and False otherwise :type stream: bool + :param raise_invalid_id: If True an InvalidInstanceId exception will be raised in case service returns HTTP 400 + :type raise_invalid_id: bool + :param ignore_exception: If True no error messages will be shown in case of exceptions + :type ignore_exception: bool :return: download response or None if download failed :rtype: requests.response or None """ try: - response = requests.get(url, stream=stream) + response = requests.get(url, stream=stream, + headers={'User-Agent': 'sh_qgis_plugin_{}'.format(self.plugin_version)}) response.raise_for_status() except requests.RequestException as exception: - message = '{}: '.format(exception.__class__.__name__) - - if isinstance(exception, requests.ConnectionError): - message += 'Cannot access service, check your internet connection.' - elif isinstance(exception, requests.HTTPError): - try: - server_message = '' - for elem in ElementTree.fromstring(exception.response.content): - if 'ServiceException' in elem.tag: - server_message += elem.text.strip('\n\t ') - except ElementTree.ParseError: - server_message = exception.response.text.strip('\n\t ') - server_message = server_message.encode('ascii', errors='ignore').decode('utf-8') - message += 'server response: "{}"'.format(server_message) - else: - message += str(exception) + if ignore_exception: + return + if raise_invalid_id and isinstance(exception, requests.HTTPError) and exception.response.status_code == 400: + raise InvalidInstanceId() - self.show_message(message, CRITICAL_MSG) + self.show_message(self.get_error_message(exception), CRITICAL_MSG) response = None return response + + @staticmethod + def get_error_message(exception): + """ Creates an error message from the given exception + + :param exception: Exception obtained during download + :type exception: requests.RequestException + :return: error message + :rtype: str + """ + message = '{}: '.format(exception.__class__.__name__) + + if isinstance(exception, requests.ConnectionError): + return message + 'Cannot access service, check your internet connection.' + + if isinstance(exception, requests.HTTPError): + try: + server_message = '' + for elem in ElementTree.fromstring(exception.response.content): + if 'ServiceException' in elem.tag: + server_message += elem.text.strip('\n\t ') + except ElementTree.ParseError: + server_message = exception.response.text.strip('\n\t ') + server_message = server_message.encode('ascii', errors='ignore').decode('utf-8') + if 'Config instance "instance.' in server_message: + instance_id = server_message.split('"')[1][9:] + server_message = 'Invalid instance id: {}'.format(instance_id) + return message + 'server response: "{}"'.format(server_message) + + return message + str(exception) # ---------------------------------------------------------------------------- def add_wms_layer(self): @@ -425,7 +494,7 @@ def add_wms_layer(self): return self.missing_instance_id() self.update_parameters() - name = '{} - {}'.format(Settings.parameters['prettyName'], Settings.parameters['title']) + name = '{} - {}'.format(self.get_source_name(), Settings.parameters['title']) new_layer = QgsRasterLayer(self.get_wms_uri(), name, 'wms') if new_layer.isValid(): QgsProject.instance().addMapLayer(new_layer) @@ -543,15 +612,32 @@ def update_parameters(self): Update parameters from GUI :return: """ + Settings.parameters['priority'] = Settings.priority_map[self.dockwidget.priority.currentText()] + Settings.parameters['maxcc'] = str(self.dockwidget.maxcc.value()) + Settings.parameters['time'] = str(self.get_time()) + Settings.parameters['crs'] = self.dockwidget.epsg.currentText().replace(' ', '') + + self.update_selected_layer() + + def update_selected_layer(self): + """ Updates properties of selected Sentinel Hub layer + """ layers_index = self.dockwidget.layers.currentIndex() + old_data_source = self.data_source if 0 <= layers_index < len(self.capabilities): Settings.parameters['layers'] = self.capabilities[layers_index]['Name'] Settings.parameters_wcs['coverage'] = self.capabilities[layers_index]['Name'] Settings.parameters['title'] = self.capabilities[layers_index]['Title'] - Settings.parameters['priority'] = self.dockwidget.priority.currentText() - Settings.parameters['maxcc'] = str(self.dockwidget.maxcc.value()) - Settings.parameters['time'] = str(self.get_time()) - Settings.parameters['crs'] = self.dockwidget.epsg.currentText().replace(' ', '') + + if self.base_url in [Settings.services_base_url, Settings.uswest_base_url]: + self.data_source = self.capabilities[layers_index].get('Dataset') + if self.data_source: + self.base_url = Settings.data_source_props[self.data_source]['url'] + Settings.parameters_wfs['typenames'] = Settings.data_source_props[self.data_source]['wfs_name'] + + # TODO: if DEM, disable times + if old_data_source != self.data_source: + self.get_cloud_cover() def update_maxcc_label(self): """ @@ -660,19 +746,29 @@ def download_caption(self): self.download_wcs_data(url, filename) - @staticmethod - def get_filename(bbox): + def get_filename(self, bbox): """ Prepare filename which contains some metadata - sentinel2_LAYER_time0_time1_xmin_y_min_xmax_ymax_maxcc_priority.FORMAT + DataSource_LayerName_time0_time1_xmin_y_min_xmax_ymax_maxcc_priority.FORMAT :param bbox: :return: """ - info_list = [Settings.parameters['name'], Settings.parameters['layers']] \ + info_list = [self.get_source_name(), Settings.parameters['layers']] \ + Settings.parameters['time'].split('/')[:2] + bbox.split(',') \ + [Settings.parameters['maxcc'], Settings.parameters['priority']] - return '.'.join(map(str, ['_'.join(map(str, info_list)), + name = '.'.join(map(str, ['_'.join(map(str, info_list)), Settings.parameters_wcs['format'].split(';')[0].split('/')[1]])) + return name.replace(' ', '').replace(':', '_') + + def get_source_name(self): + """ Returns name of the data source + + :return: name + :rtype: string + """ + if self.base_url == Settings.ipt_base_url: + return 'EO Cloud' + return Settings.data_source_props[self.data_source]['pretty_name'] def update_maxcc(self): """ @@ -702,7 +798,7 @@ def change_exact_date(self): def change_instance_id(self): """ - Change Instance ID, and validate that is valid + Change Instance ID, and check that it is valid :return: """ new_instance_id = self.dockwidget.instanceId.text() @@ -721,6 +817,8 @@ def change_instance_id(self): if self.instance_id: self.show_message("New Instance ID and layers set.", SUCCESS_MSG) QSettings().setValue(Settings.instance_id_location, new_instance_id) + self.update_parameters() + self.get_cloud_cover() else: self.dockwidget.instanceId.setText(self.instance_id) @@ -744,14 +842,16 @@ def update_month(self): :return: """ self.update_parameters() + self.get_cloud_cover() + def get_calendar_month_interval(self): year = self.dockwidget.calendar.yearShown() month = self.dockwidget.calendar.monthShown() _, number_of_days = calendar.monthrange(year, month) first = datetime.date(year, month, 1) last = datetime.date(year, month, number_of_days) - self.get_cloud_cover(first.strftime('%Y-%m-%d') + '/' + last.strftime('%Y-%m-%d') + '/P1D') + return '{}/{}/P1D'.format(first.strftime('%Y-%m-%d'), last.strftime('%Y-%m-%d')) def toggle_extent(self, setting): """ @@ -796,7 +896,7 @@ def get_values(self): try: float(value) except ValueError: - return None + return return new_values def run(self): @@ -823,6 +923,7 @@ def run(self): self.dockwidget.selectDestination.clicked.connect(self.select_destination) # Render input fields changes and events + self.dockwidget.layers.currentIndexChanged.connect(self.update_selected_layer) self.dockwidget.time0.selectionChanged.connect(lambda: self.move_calendar('time0')) self.dockwidget.time1.selectionChanged.connect(lambda: self.move_calendar('time1')) self.dockwidget.calendar.clicked.connect(self.add_time) diff --git a/SentinelHub/SentinelHub_dockwidget_base.ui b/SentinelHub/SentinelHub_dockwidget_base.ui index 64bcf09..250cd92 100644 --- a/SentinelHub/SentinelHub_dockwidget_base.ui +++ b/SentinelHub/SentinelHub_dockwidget_base.ui @@ -905,7 +905,7 @@ - Cloud coverage 20% + Cloud coverage 100% -1 @@ -930,10 +930,10 @@ 100 - 20 + 100 - 20 + 100 Qt::Horizontal diff --git a/SentinelHub/Settings.py b/SentinelHub/Settings.py index ebc9a81..d663a4c 100644 --- a/SentinelHub/Settings.py +++ b/SentinelHub/Settings.py @@ -1,31 +1,37 @@ #!/usr/bin/env python # encoding: utf-8 +""" +This script contains all +""" -# Base setup values -url_base = 'https://services.sentinel-hub.com/ogc/' +# Base url +services_base_url = 'https://services.sentinel-hub.com/ogc/' +uswest_base_url = 'https://services-uswest2.sentinel-hub.com/ogc/' +ipt_base_url = 'http://services.eocloud.sentinel-hub.com/v1/' +# Locations where QGIS will save values instance_id_location = "SentinelHub/instance_id" download_folder_location = "SentinelHub/download_folder" +# Supported CRS epsg = [ 'EPSG: 3857', 'EPSG: 4326' ] -# Request parameters +# Main request parameters parameters = { - 'name': 'sentinel2', - 'prettyName': 'Sentinel 2', 'title': '', 'showLogo': 'false', 'layers': '', - 'maxcc': '20', + 'maxcc': '100', 'priority': 'mostRecent', 'time': '', 'crs': 'EPSG:3857' } -parameters_wms = { # The first 3 parameters are required for qgis layer +# WMS parameters - the first 3 parameters are required for qgis layer +parameters_wms = { 'IgnoreGetFeatureInfoUrl': '1', 'IgnoreGetMapUrl': '1', 'contextualWMSLegend': '0', @@ -37,6 +43,7 @@ 'version': '1.3.0', } +# WFS parameters parameters_wfs = { 'service': 'WFS', 'version': '2.0.0', @@ -46,6 +53,7 @@ 'outputformat': 'application/json', } +# WCS parameters parameters_wcs = { 'service': 'wcs', 'request': 'GetCoverage', @@ -56,8 +64,29 @@ 'resy': '10' } -# enum values of parameters -priority_list = ['mostRecent', 'leastRecent', 'leastCC'] +data_source_props = {'S2L1C': {'url': services_base_url, + 'wfs_name': 'S2.TILE', + 'pretty_name': 'Sentinel-2 L1C'}, + 'S2L2A': {'url': services_base_url, + 'wfs_name': 'DSS2', + 'pretty_name': 'Sentinel-2 L2A'}, + 'S1GRD': {'url': services_base_url, + 'wfs_name': 'DSS3', + 'pretty_name': 'Sentinel-1'}, + 'L8L1C': {'url': uswest_base_url, + 'wfs_name': 'DSS6', + 'pretty_name': 'Landsat 8'}, + 'MODIS': {'url': uswest_base_url, + 'wfs_name': 'DSS5', + 'pretty_name': 'MODIS'}, + 'DEM': {'url': uswest_base_url, + 'wfs_name': 'DSS4', + 'pretty_name': 'DEM'}} + +# values for UI selections +priority_map = {'Most recent': 'mostRecent', + 'Least recent': 'leastRecent', + 'Least cloud coverage': 'leastCC'} atmfilter_list = ['NONE', 'DOS1', 'ATMCOR'] cloud_correction = ['NONE', 'REPLACE'] img_formats = ['image/png', 'image/jpeg', diff --git a/SentinelHub/metadata.txt b/SentinelHub/metadata.txt index 316ece4..df88745 100644 --- a/SentinelHub/metadata.txt +++ b/SentinelHub/metadata.txt @@ -1,12 +1,6 @@ -# This file contains metadata for your plugin. Since -# version 2.0 of QGIS this is the proper way to supply -# information about a plugin. The old method of -# embedding metadata in __init__.py -# is no longer supported since version 2.0. +# This file contains metadata for your plugin. -# This file should be included when you package your plugin. # Mandatory items: - [general] name=SentinelHub qgisMinimumVersion=2.0 @@ -30,7 +24,7 @@ repository=https://github.com/sinergise/qgis_sentinel_hub.git # changelog= # Tags are comma separated with spaces allowed -tags=SentinelHub, Sinergise, remote sensing, satellite, images, viewer, download, Sentinel, Landsat, Modis +tags=SentinelHub, Sinergise, remote sensing, satellite, images, viewer, download, Sentinel, Landsat, Modis, DEM, Envisat category=Web icon=favicon.ico