From af2caf1a843ec83d2f4041b778904b7000f0d55d Mon Sep 17 00:00:00 2001 From: Matic Lubej Date: Wed, 22 Jan 2020 14:16:38 +0100 Subject: [PATCH 01/11] updating eoexecution logging level --- core/eolearn/core/eoexecution.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/eolearn/core/eoexecution.py b/core/eolearn/core/eoexecution.py index 43d09490d..72477d11d 100644 --- a/core/eolearn/core/eoexecution.py +++ b/core/eolearn/core/eoexecution.py @@ -171,13 +171,13 @@ def _try_add_logging(cls, log_path): def _try_remove_logging(cls, log_path, logger, handler, stats): if log_path: try: - logger.info(msg='Pipeline failed.' if cls.STATS_ERROR in stats else 'Pipeline finished.') + message = 'EOWorkflow execution {}'.format('failed' if cls.STATS_ERROR in stats else 'finished') + logger.debug(message) handler.close() logger.removeHandler(handler) except BaseException: pass - @classmethod def _execute_workflow(cls, process_args): """ Handles a single execution of a workflow From b4ac31f7b2ee70feae86c5bf29c1cb6cfbe3ee6f Mon Sep 17 00:00:00 2001 From: AleksMat Date: Wed, 22 Jan 2020 16:27:01 +0100 Subject: [PATCH 02/11] fixed a bug where LinearWorkflow loses task name and added a unit test that checks that --- core/eolearn/core/eoworkflow.py | 2 +- core/eolearn/tests/test_eoworkflow.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/core/eolearn/core/eoworkflow.py b/core/eolearn/core/eoworkflow.py index 6d2efb210..7470ba778 100644 --- a/core/eolearn/core/eoworkflow.py +++ b/core/eolearn/core/eoworkflow.py @@ -327,7 +327,7 @@ def __init__(self, *tasks, **kwargs): tasks = [self._parse_task(task) for task in tasks] tasks = self._make_tasks_unique(tasks) - dependencies = [(task, [tasks[idx - 1][0]] if idx > 0 else []) for idx, (task, name) in enumerate(tasks)] + dependencies = [(task, [tasks[idx - 1][0]] if idx > 0 else [], name) for idx, (task, name) in enumerate(tasks)] super().__init__(dependencies, **kwargs) @staticmethod diff --git a/core/eolearn/tests/test_eoworkflow.py b/core/eolearn/tests/test_eoworkflow.py index b5c28cdc3..af6a7b823 100644 --- a/core/eolearn/tests/test_eoworkflow.py +++ b/core/eolearn/tests/test_eoworkflow.py @@ -108,9 +108,10 @@ def test_workflow_arguments(self): def test_linear_workflow(self): in_task = InputTask() + in_task_name = 'My input task' inc_task = Inc() pow_task = Pow() - eow = LinearWorkflow((in_task, 'task name'), inc_task, inc_task, pow_task) + eow = LinearWorkflow((in_task, in_task_name), inc_task, inc_task, pow_task) res = eow.execute({ in_task: {'val': 2}, inc_task: {'d': 2}, # Note that this will assign value only to one instance of Inc task @@ -118,6 +119,11 @@ def test_linear_workflow(self): }) self.assertEqual(res[pow_task], (2 + 2 + 1) ** 3) + task_map = eow.get_tasks() + self.assertTrue(in_task_name in task_map, "A task with name '{}' should be amongst tasks".format(in_task_name)) + self.assertEqual(task_map[in_task_name], in_task, + "A task with name '{}' should map into {}".format(in_task_name, in_task)) + def test_get_tasks(self): in_task = InputTask() inc_task = Inc() From 739281f02aad600919b4e6962b6c08e4f506f69d Mon Sep 17 00:00:00 2001 From: AleksMat Date: Wed, 22 Jan 2020 16:48:23 +0100 Subject: [PATCH 03/11] fixed EOWorkflow.get_tasks method in case task names are duplicated --- core/eolearn/core/eoworkflow.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/core/eolearn/core/eoworkflow.py b/core/eolearn/core/eoworkflow.py index 7470ba778..8bfba57f3 100644 --- a/core/eolearn/core/eoworkflow.py +++ b/core/eolearn/core/eoworkflow.py @@ -268,14 +268,25 @@ def get_tasks(self): """ Returns an ordered dictionary {task_name: task} of all tasks within this workflow :return: Ordered dictionary with key being task_name (str) and an instance of a corresponding task from this - workflow + workflow :rtype: OrderedDict """ - tasks = collections.OrderedDict() + task_dict = collections.OrderedDict() for dep in self.ordered_dependencies: - tasks[dep.name] = dep.task + task_name = dep.name - return tasks + if task_name in task_dict: + new_task_name = task_name + count = 0 + while new_task_name in task_dict: + count += 1 + new_task_name = '{}({})'.format(task_name, count) + + task_name = new_task_name + + task_dict[task_name] = dep.task + + return task_dict def get_dot(self): """ Generates the DOT description of the underlying computational graph From a51009230ba17528e4462fe022af5f2c92bdae15 Mon Sep 17 00:00:00 2001 From: AleksMat Date: Wed, 22 Jan 2020 17:05:34 +0100 Subject: [PATCH 04/11] improved unit tests for EOWorkflow.get_tasks() --- core/eolearn/core/eoworkflow.py | 2 +- core/eolearn/tests/test_eoworkflow.py | 24 +++++++++--------------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/core/eolearn/core/eoworkflow.py b/core/eolearn/core/eoworkflow.py index 8bfba57f3..b322f0719 100644 --- a/core/eolearn/core/eoworkflow.py +++ b/core/eolearn/core/eoworkflow.py @@ -268,7 +268,7 @@ def get_tasks(self): """ Returns an ordered dictionary {task_name: task} of all tasks within this workflow :return: Ordered dictionary with key being task_name (str) and an instance of a corresponding task from this - workflow + workflow. The order of tasks is the same as in which they will be executed. :rtype: OrderedDict """ task_dict = collections.OrderedDict() diff --git a/core/eolearn/tests/test_eoworkflow.py b/core/eolearn/tests/test_eoworkflow.py index af6a7b823..bceef7f76 100644 --- a/core/eolearn/tests/test_eoworkflow.py +++ b/core/eolearn/tests/test_eoworkflow.py @@ -127,35 +127,29 @@ def test_linear_workflow(self): def test_get_tasks(self): in_task = InputTask() inc_task = Inc() - pow_task = Pow() - task_names = ['InputTask', 'Inc', 'Pow'] - workflow_tasks = [in_task, inc_task, pow_task] - eow = LinearWorkflow(*workflow_tasks) + task_names = ['InputTask', 'Inc', 'Inc(1)', 'Inc(2)'] + eow = LinearWorkflow(in_task, inc_task, inc_task, inc_task) returned_tasks = eow.get_tasks() # check if tasks are present - for task_name in task_names: - self.assertIn(task_name, returned_tasks.keys()) + self.assertEqual(sorted(task_names), sorted(returned_tasks)) # check if tasks still work arguments_dict = { in_task: {'val': 2}, - inc_task: {'d': 2}, - pow_task: {'n': 3} + inc_task: {'d': 2} } res_workflow = eow.execute(arguments_dict) - res_workflow_value = [res_workflow[key] for key in res_workflow.keys()][0] + res_workflow_value = list(res_workflow.values()) - for idx, task in enumerate(workflow_tasks): - if idx == 0: - res_tasks_value = task.execute(**arguments_dict[task]) - else: - res_tasks_value = task.execute(res_tasks_value, **arguments_dict[task]) + res_tasks_values = [] + for idx, task in enumerate(returned_tasks.values()): + res_tasks_values = [task.execute(*res_tasks_values, **arguments_dict.get(task, {}))] - self.assertEqual(res_workflow_value, res_tasks_value) + self.assertEqual(res_workflow_value, res_tasks_values) def test_trivial_workflow(self): task = DummyTask() From d2815f8b4ed1b983663c9a0e14161247997d7623 Mon Sep 17 00:00:00 2001 From: AleksMat Date: Wed, 22 Jan 2020 17:15:28 +0100 Subject: [PATCH 05/11] move custom task generation to Dependency object --- core/eolearn/core/eoworkflow.py | 13 +++++++++---- core/eolearn/tests/test_eoworkflow.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/core/eolearn/core/eoworkflow.py b/core/eolearn/core/eoworkflow.py index b322f0719..dc2c6de54 100644 --- a/core/eolearn/core/eoworkflow.py +++ b/core/eolearn/core/eoworkflow.py @@ -276,13 +276,11 @@ def get_tasks(self): task_name = dep.name if task_name in task_dict: - new_task_name = task_name count = 0 - while new_task_name in task_dict: + while dep.get_custom_name(count) in task_dict: count += 1 - new_task_name = '{}({})'.format(task_name, count) - task_name = new_task_name + task_name = dep.get_custom_name(count) task_dict[task_name] = dep.task @@ -407,6 +405,13 @@ def set_name(self, name): """ self.name = name + def get_custom_name(self, number=0): + """ Provides custom task name according to given number. E.g. FooTask -> FooTask + """ + if number: + return '{}_{}'.format(self.name, number) + return self.name + class WorkflowResults(collections.Mapping): """ The result of a workflow is an (immutable) dictionary mapping [1] from EOTasks to results of the workflow. diff --git a/core/eolearn/tests/test_eoworkflow.py b/core/eolearn/tests/test_eoworkflow.py index bceef7f76..75d707378 100644 --- a/core/eolearn/tests/test_eoworkflow.py +++ b/core/eolearn/tests/test_eoworkflow.py @@ -128,7 +128,7 @@ def test_get_tasks(self): in_task = InputTask() inc_task = Inc() - task_names = ['InputTask', 'Inc', 'Inc(1)', 'Inc(2)'] + task_names = ['InputTask', 'Inc', 'Inc_1', 'Inc_2'] eow = LinearWorkflow(in_task, inc_task, inc_task, inc_task) returned_tasks = eow.get_tasks() From 4deff500ea830653164b73d79e52bcb239ce71a6 Mon Sep 17 00:00:00 2001 From: AleksMat Date: Wed, 22 Jan 2020 17:25:14 +0100 Subject: [PATCH 06/11] used Dependency.get_custom_name also for workflow visualization --- .../eolearn/visualization/eoworkflow_visualization.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/visualization/eolearn/visualization/eoworkflow_visualization.py b/visualization/eolearn/visualization/eoworkflow_visualization.py index c8faf90d6..d1aa6be2f 100644 --- a/visualization/eolearn/visualization/eoworkflow_visualization.py +++ b/visualization/eolearn/visualization/eoworkflow_visualization.py @@ -73,11 +73,7 @@ def _get_dep_to_dot_name_mapping(dependencies): dep_to_dot_name = {} for dot_name, deps in dot_name_to_deps.items(): - if len(deps) == 1: - dep_to_dot_name[deps[0]] = dot_name - continue - for idx, dep in enumerate(deps): - dep_to_dot_name[dep] = dot_name + str(idx) + dep_to_dot_name[dep] = dep.get_custom_name(idx) return dep_to_dot_name From 20d2a60a4dbbeeb9cdbdf9229a4b2b4949de8e89 Mon Sep 17 00:00:00 2001 From: iovsn Date: Wed, 5 Feb 2020 08:52:06 +0100 Subject: [PATCH 07/11] Added processing api evalscript version --- io/eolearn/io/processing_api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/io/eolearn/io/processing_api.py b/io/eolearn/io/processing_api.py index 7708467ee..59ef3964d 100644 --- a/io/eolearn/io/processing_api.py +++ b/io/eolearn/io/processing_api.py @@ -206,6 +206,8 @@ def generate_evalscript(self): """ Generate the evalscript to be passed with the request, based on chosen bands """ evalscript = """ + //VERSION=3 + function setup() {{ return {{ input: [{{ @@ -363,6 +365,8 @@ def _build_payloads(self, bbox, size_x, size_y, timestamp, time_interval): """ Build payloads for the requests to the service """ evalscript = """ + //VERSION=3 + function setup() { return { input: ["DEM"], From b7ddbd3f9a71f6c96264dfc2f45977f34c5e9dab Mon Sep 17 00:00:00 2001 From: iovsn Date: Wed, 5 Feb 2020 09:47:33 +0100 Subject: [PATCH 08/11] Version 0.7.1 (core, io, visualization update) --- core/eolearn/core/__init__.py | 2 +- io/eolearn/io/__init__.py | 2 +- setup.py | 8 ++++---- visualization/eolearn/visualization/__init__.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/eolearn/core/__init__.py b/core/eolearn/core/__init__.py index eb95a305c..c04dff750 100644 --- a/core/eolearn/core/__init__.py +++ b/core/eolearn/core/__init__.py @@ -18,4 +18,4 @@ from .utilities import deep_eq, negate_mask, constant_pad, get_common_timestamps, bgr_to_rgb, FeatureParser -__version__ = '0.7.0' +__version__ = '0.7.1' diff --git a/io/eolearn/io/__init__.py b/io/eolearn/io/__init__.py index d9fc97447..ec77f2724 100644 --- a/io/eolearn/io/__init__.py +++ b/io/eolearn/io/__init__.py @@ -9,4 +9,4 @@ from .local_io import ExportToTiff, ImportFromTiff from .processing_api import SentinelHubInputTask, SentinelHubDemTask -__version__ = '0.7.0' +__version__ = '0.7.1' diff --git a/setup.py b/setup.py index f04fb9f56..c1684690b 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ def parse_requirements(file): setup( name='eo-learn', python_requires='>=3.5', - version='0.7.0', + version='0.7.1', description='Earth observation processing framework for machine learning in Python', long_description=get_long_description(), long_description_content_type='text/markdown', @@ -33,14 +33,14 @@ def parse_requirements(file): packages=[], include_package_data=True, install_requires=[ - 'eo-learn-core>=0.7.0', + 'eo-learn-core>=0.7.1', 'eo-learn-coregistration>=0.7.0', 'eo-learn-features>=0.7.0', 'eo-learn-geometry>=0.7.0', - 'eo-learn-io>=0.7.0', + 'eo-learn-io>=0.7.1', 'eo-learn-mask>=0.7.0', 'eo-learn-ml-tools>=0.7.0', - 'eo-learn-visualization>=0.7.0' + 'eo-learn-visualization>=0.7.1' ], extras_require={ 'DEV': parse_requirements('requirements-dev.txt') diff --git a/visualization/eolearn/visualization/__init__.py b/visualization/eolearn/visualization/__init__.py index ee77669b4..33bf3bd4c 100644 --- a/visualization/eolearn/visualization/__init__.py +++ b/visualization/eolearn/visualization/__init__.py @@ -10,4 +10,4 @@ except ImportError: pass -__version__ = '0.7.0' +__version__ = '0.7.1' From 76c35717542d9aa31f7ff73ff36244381e7a6c4a Mon Sep 17 00:00:00 2001 From: iovsn Date: Wed, 5 Feb 2020 10:44:46 +0100 Subject: [PATCH 09/11] Fixed processing api docs --- docs/source/eolearn.io.processing_api.rst | 7 +++++++ docs/source/eolearn.io.rst | 1 + 2 files changed, 8 insertions(+) create mode 100644 docs/source/eolearn.io.processing_api.rst diff --git a/docs/source/eolearn.io.processing_api.rst b/docs/source/eolearn.io.processing_api.rst new file mode 100644 index 000000000..4e0569f94 --- /dev/null +++ b/docs/source/eolearn.io.processing_api.rst @@ -0,0 +1,7 @@ +eolearn.io.processing\_api +=============================== + +.. automodule:: eolearn.io.processing_api + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/eolearn.io.rst b/docs/source/eolearn.io.rst index 1c4484849..05bed8484 100644 --- a/docs/source/eolearn.io.rst +++ b/docs/source/eolearn.io.rst @@ -13,3 +13,4 @@ Submodules: eolearn.io.geopedia eolearn.io.local_io eolearn.io.sentinelhub_service + eolearn.io.processing_api From 1c64dba49ffb06d47a0de9011af1b634eb41017d Mon Sep 17 00:00:00 2001 From: iovsn Date: Wed, 5 Feb 2020 10:52:26 +0100 Subject: [PATCH 10/11] Added CHANGELOG.md --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..e5955db33 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ +## [Unreleased] + +## [Version 0.7.1] - 2019-02-05 +### Fixed +- `eolearn.io.SentinelHubInputTask`: evalscript version was not passed to the sentinel-hub service. +- `eolearn.core.EOWorkflow`: fixed generating task dependencies. +### Added +- Processing API docs generation. \ No newline at end of file From 7f09edf1bf5916754f48e3759d11924368971607 Mon Sep 17 00:00:00 2001 From: iovsn Date: Wed, 5 Feb 2020 10:53:39 +0100 Subject: [PATCH 11/11] Update changelog. --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5955db33..0511dd7ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,4 +5,5 @@ - `eolearn.io.SentinelHubInputTask`: evalscript version was not passed to the sentinel-hub service. - `eolearn.core.EOWorkflow`: fixed generating task dependencies. ### Added -- Processing API docs generation. \ No newline at end of file +- Processing API docs generation. +- Introduced CHANGELOG.md. \ No newline at end of file