From cbab884b30c2aaf01b1d8a9bfdca66267547125c Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Fri, 20 Dec 2019 14:37:59 +0100 Subject: [PATCH 01/51] added filename and funcName to logging messages --- mapswipe_workers/config/logging.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/config/logging.cfg b/mapswipe_workers/config/logging.cfg index a3696b031..30cd81d2e 100644 --- a/mapswipe_workers/config/logging.cfg +++ b/mapswipe_workers/config/logging.cfg @@ -30,4 +30,4 @@ formatter=mapswipeFormatter args=('/var/log/mapswipe_workers/mapswipe_workers.log','D', 1, 14,) [formatter_mapswipeFormatter] -format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +format=%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(message)s From 29847abbf3ef39389a30ef88f87050bde554d3f8 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Sat, 28 Dec 2019 10:12:36 +0100 Subject: [PATCH 02/51] run black and flake8 --- .../firebase_to_postgres/update_data.py | 39 ++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/update_data.py b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/update_data.py index bc68aa9ad..06bacec38 100644 --- a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/update_data.py +++ b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/update_data.py @@ -4,10 +4,8 @@ from mapswipe_workers.definitions import logger -def update_user_data(user_ids=None): - """ - Copies new users from Firebase to Postgres - """ +def update_user_data(user_ids: list = []): + """Copies new users from Firebase to Postgres.""" # TODO: On Conflict fb_db = auth.firebaseDB() pg_db = auth.postgresDB() @@ -24,15 +22,11 @@ def update_user_data(user_ids=None): try: last_updated = last_updated[0][0] logger.info(f"got last updated timestamp: {last_updated}") - except: + except IndexError: logger.info("could not get last timestamp") - last_updated = None + last_updated = [] - if last_updated is None: - # No users in the Postgres database yet. - # Get all users from Firebase. - users = fb_ref.get() - else: + if last_updated: # Get only new users from Firebase. last_updated = last_updated.strftime("%Y-%m-%dT%H:%M:%S.%fZ") fb_query = fb_ref.order_by_child("created").start_at(last_updated) @@ -40,15 +34,17 @@ def update_user_data(user_ids=None): # Delete first user in ordered dict. # This user is already in the database (user.created = last_updated). if len(users) == 0: - logger.info( - f"there are no new users in firebase based on created timestamp" - ) + logger.info("there are no new users in firebase based on created timestamp") else: users.popitem(last=False) + else: + # No users in the Postgres database yet. + # Get all users from Firebase. + users = fb_ref.get() # update users specified in user_ids list if user_ids: - logger.info(f"will add users to copy_new_users based on user_ids provided") + logger.info("will add users to copy_new_users based on user_ids provided") for user_id in user_ids: user = fb_ref.child(user_id).get() users[user_id] = user @@ -65,7 +61,8 @@ def update_user_data(user_ids=None): # if user has no "created" attribute, we set it to current time created = dt.datetime.utcnow().isoformat()[0:-3] + "Z" logger.info( - f"user {user_id} didn't have a 'created' attribute. Set it to '{created}' now." + f"user {user_id} didn't have a 'created' attribute. " + f"Set it to '{created}' now." ) try: @@ -74,7 +71,8 @@ def update_user_data(user_ids=None): # if user has no "username" attribute, we set it to None username = None logger.info( - f"user {user_id} didn't have a 'username' attribute. Set it to '{username}' now." + f"user {user_id} didn't have a 'username' attribute. " + f"Set it to '{username}' now." ) query_update_user = """ @@ -98,17 +96,14 @@ def update_user_data(user_ids=None): logger.info("Updated user data in Potgres") -def update_project_data(project_ids=None): +def update_project_data(project_ids: list = []): """ Gets status of projects from Firebase and updates them in Postgres. + Default behavior is to update all projects. If called with a list of project ids as parameter only those projects will be updated. - - Parameters - ---------- - project_ids: list """ fb_db = auth.firebaseDB() From 7d5ce28b832077fe13d3035467885723294a3548 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Sat, 28 Dec 2019 10:48:19 +0100 Subject: [PATCH 03/51] add setup.cfg for python plugin (flake8) and util confguration --- mapswipe_workers/setup.cfg | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 mapswipe_workers/setup.cfg diff --git a/mapswipe_workers/setup.cfg b/mapswipe_workers/setup.cfg new file mode 100644 index 000000000..0c935431c --- /dev/null +++ b/mapswipe_workers/setup.cfg @@ -0,0 +1,18 @@ +# All configuration for plugins and other utils is defined here. +# Read more about `setup.cfg`: +# https://docs.python.org/3/distutils/configfile.html + + +[flake8] +# W503: https://github.com/python/black#line-breaks--binary-operators +# max-line-lenght conform with Black +ignore = "W503" +inline-quotes = double +max-line-length = 88 + + +[isort] +# https://github.com/timothycrosley/isort/wiki/isort-Settings +# See https://github.com/timothycrosley/isort#multi-line-output-modes +include_trailing_comma = true +multi_line_output = 3 From 7055924adf497da206f591d33150877126f8a354 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Sat, 28 Dec 2019 10:49:32 +0100 Subject: [PATCH 04/51] remove arguments of flake8 which are already defined in setup.cfg --- mapswipe_workers/.pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/mapswipe_workers/.pre-commit-config.yaml b/mapswipe_workers/.pre-commit-config.yaml index 875c5acd0..759c36040 100644 --- a/mapswipe_workers/.pre-commit-config.yaml +++ b/mapswipe_workers/.pre-commit-config.yaml @@ -7,4 +7,3 @@ repos: rev: 3.7.9 hooks: - id: flake8 - args: ["--ignore=E501,W503"] From 20ca771f498ccdd5ccb7994e5fe0cb80a1f94e9e Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Sat, 28 Dec 2019 10:51:21 +0100 Subject: [PATCH 05/51] add setup.cfg for python plugin (flake8) and util confguration --- mapswipe_workers/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/setup.cfg b/mapswipe_workers/setup.cfg index 0c935431c..0cdb1de72 100644 --- a/mapswipe_workers/setup.cfg +++ b/mapswipe_workers/setup.cfg @@ -5,7 +5,7 @@ [flake8] # W503: https://github.com/python/black#line-breaks--binary-operators -# max-line-lenght conform with Black +# max-line-lenght = 88 to conform with Black ignore = "W503" inline-quotes = double max-line-length = 88 From 8aad6282d0fc803e0a5b79e864dce0b258c4a7e5 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Sat, 28 Dec 2019 19:43:57 +0100 Subject: [PATCH 06/51] Add archive_project command. --- .../mapswipe_workers/mapswipe_workers.py | 47 +++++++++++++------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index b63a1e2c9..05d776612 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -1,28 +1,28 @@ +import ast +import json import time + import click import schedule as sched -import json -import ast - -from mapswipe_workers.definitions import CustomError -from mapswipe_workers.definitions import logger -from mapswipe_workers.utils import slack -from mapswipe_workers.utils import sentry from mapswipe_workers import auth +from mapswipe_workers.definitions import CustomError, logger +from mapswipe_workers.firebase_to_postgres import ( + archive_project, + transfer_results, + update_data, +) from mapswipe_workers.generate_stats import generate_stats -from mapswipe_workers.firebase_to_postgres import transfer_results -from mapswipe_workers.firebase_to_postgres import update_data +from mapswipe_workers.project_types.build_area import build_area_tutorial from mapswipe_workers.project_types.build_area.build_area_project import ( BuildAreaProject, ) -from mapswipe_workers.project_types.build_area import build_area_tutorial -from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject +from mapswipe_workers.project_types.change_detection import change_detection_tutorial from mapswipe_workers.project_types.change_detection.change_detection_project import ( ChangeDetectionProject, ) -from mapswipe_workers.project_types.change_detection import change_detection_tutorial -from mapswipe_workers.utils import user_management +from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject +from mapswipe_workers.utils import sentry, slack, user_management class PythonLiteralOption(click.Option): @@ -294,6 +294,22 @@ def run_create_tutorial(input_file): logger.exception(e) +@click.command("archive") +@click.option( + "--project-id", + "-i", + help=( + "Archive project in Postgres. " + + "Delete groups, tasks and results from Firebase." + ), + type=str, +) +def run_archive_project(project_id): + update_data.update_project_data([project_id]) + transfer_results.transfer_results([project_id]) + archive_project.archive_project(project_id) + + @click.command("run") @click.option( "--schedule", @@ -445,10 +461,11 @@ def _run_user_management(email, manager): user_management.remove_project_manager_rights(email) +cli.add_command(run) +cli.add_command(run_archive_project) cli.add_command(run_create_projects) +cli.add_command(run_create_tutorial) cli.add_command(run_firebase_to_postgres) cli.add_command(run_generate_stats) cli.add_command(run_generate_stats_all_projects) cli.add_command(run_user_management) -cli.add_command(run_create_tutorial) -cli.add_command(run) From 8c593254d834e5564a1837cba13849a0a4da46ae Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Sat, 28 Dec 2019 19:45:01 +0100 Subject: [PATCH 07/51] Add archive feature. Archive project in Postgres. Delete project in Firebase. --- .../firebase_to_postgres/archive_project.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py diff --git a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py new file mode 100644 index 000000000..b4c6ec3e7 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py @@ -0,0 +1,37 @@ +""" +Archive a project. + +Delete groups, tasks and results of a project in Firebase. +Keep all informations in Postgres. +""" + +from mapswipe_workers import auth +from mapswipe_workers.definitions import logger + + +def archive_project(project_id: str) -> None: + """ + Archive a project. + + Call a function which deletes groups, tasks and results in Firebase. + Set archive = true for project in Postgres. + """ + logger.info("Archive project with the id {0}".format(project_id)) + delete_project_from_firebase(project_id) + + pg_db = auth.postgresDB() + sql_query = "UPDATE projects SET archived = true WHERE project_id = {0}".format( + project_id + ) + pg_db.query(sql_query) + + +def delete_project_from_firebase(project_id: str) -> None: + """Delete results, groups and tasks of given project in Firebase.""" + logger.info( + "Delete results, groups and tasks of project with the id {0}".format(project_id) + ) + fb_db = auth.firebaseDB() + fb_db.reference("v2/results/{0}".format(project_id)).set({}) + fb_db.reference("v2/groups/{0}".format(project_id)).set({}) + fb_db.reference("v2/tasks/{0}".format(project_id)).set({}) From 7c0b3a7736541c93fd32bff0b6eef238f9418326 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Sun, 29 Dec 2019 10:34:28 +0100 Subject: [PATCH 08/51] test template for archive_project module --- .../tests/test_archive_project.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 mapswipe_workers/tests/test_archive_project.py diff --git a/mapswipe_workers/tests/test_archive_project.py b/mapswipe_workers/tests/test_archive_project.py new file mode 100644 index 000000000..a8b350432 --- /dev/null +++ b/mapswipe_workers/tests/test_archive_project.py @@ -0,0 +1,50 @@ +import unittest + +from mapswipe_workers import auth +from mapswipe_workers.firebase_to_postgres import archive_project + + +class TestArchiveProject(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Set up before tests are run.""" + cls.project_id = "test_archive_project" + # TODO: Create project + archive_project.archive_project(cls.project_id) + + @classmethod + def tearDownClass(cls): + """Tear down after tests are run.""" + pg_db = auth.postgresDB() + sql_query = "".format(cls.project_id) + pg_db.query(sql_query) + + fb_db = auth.firebaseDB() + ref = fb_db.reference("v2/projects/{0}".format(cls.project_id)) + ref.set({}) + + def test_firebase_changes(self): + """Test if groups, tasks and results are deleted from Firebase.""" + fb_db = auth.firebaseDB() + + ref = fb_db.reference("v2/groups/{0}".format(self.project_id)) + self.assertFalse(ref.get()) + + ref = fb_db.reference("v2/tasks/{0}".format(self.project_id)) + self.assertFalse(ref.get()) + + ref = fb_db.reference("v2/results/{0}".format(self.project_id)) + self.assertFalse(ref.get()) + + def test_postgres_changes(self): + """Test if postgres project is archived.""" + pg_db = auth.postgresDB() + sql_query = "SELECT archived FROM projects WHERE project_id = {}".format( + self.project_id + ) + result = pg_db.retr_query(sql_query) + self.assertEqual(result, "archived") + + +if __name__ == "__main__": + unittest.main() From e5769fb5be530df6435a4a79ece82872538f2615 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Sun, 29 Dec 2019 22:19:40 +0100 Subject: [PATCH 09/51] add projectTypeName as class attribute --- .../mapswipe_workers/definitions.py | 63 ++++++++++--------- .../build_area/build_area_project.py | 1 + .../change_detection_project.py | 1 + .../footprint/footprint_project.py | 1 + 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index d18623167..41c7002c7 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -1,40 +1,41 @@ -import os -import json import logging import logging.config -from logging.handlers import TimedRotatingFileHandler +import os +from mapswipe_workers.project_types.build_area.build_area_project import ( + BuildAreaProject, +) +from mapswipe_workers.project_types.change_detection.change_detection_project import ( + ChangeDetectionProject, +) +from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) -CONFIG_DIR = os.path.abspath( - '/usr/share/config/mapswipe_workers/' - ) - -CONFIG_PATH = os.path.join( - CONFIG_DIR, - 'configuration.json' - ) - -SERVICE_ACCOUNT_KEY_PATH = os.path.join( - CONFIG_DIR, - 'serviceAccountKey.json' - ) - -LOGGING_CONFIG_PATH = os.path.join( - CONFIG_DIR, - 'logging.cfg' - ) - -DATA_PATH = os.path.abspath( - '/var/lib/mapswipe_workers/' - ) - -logging.config.fileConfig( - fname=LOGGING_CONFIG_PATH, - disable_existing_loggers=True - ) -logger = logging.getLogger('Mapswipe Workers') +CONFIG_DIR = os.path.abspath("/usr/share/config/mapswipe_workers/") + +CONFIG_PATH = os.path.join(CONFIG_DIR, "configuration.json") + +SERVICE_ACCOUNT_KEY_PATH = os.path.join(CONFIG_DIR, "serviceAccountKey.json") + +LOGGING_CONFIG_PATH = os.path.join(CONFIG_DIR, "logging.cfg") + +DATA_PATH = os.path.abspath("/var/lib/mapswipe_workers/") + +PROJECT_TYPE_CLASSES = { + 1: BuildAreaProject, + 2: FootprintProject, + 3: ChangeDetectionProject, +} + +PROJECT_TYPE_NAMES = { + 1: BuildAreaProject.name, + 2: FootprintProject.name, + 3: ChangeDetectionProject.name, +} + +logging.config.fileConfig(fname=LOGGING_CONFIG_PATH, disable_existing_loggers=True) +logger = logging.getLogger("Mapswipe Workers") class CustomError(Exception): diff --git a/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py b/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py index ee7363c53..2e54dd8a0 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py @@ -17,6 +17,7 @@ class BuildAreaProject(BaseProject): """ projectType = 1 + projectTypeName = "Build Area" def __init__(self, project_draft): # this will create the basis attributes diff --git a/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py b/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py index 7ce28d0fb..c8b745fac 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py @@ -19,6 +19,7 @@ class ChangeDetectionProject(BaseProject): """ projectType = 1 + projectTypeName = "Change Detection" def __init__(self, project_draft): # this will create the basis attributes diff --git a/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py b/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py index f03b299de..009ae11e1 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py @@ -17,6 +17,7 @@ class FootprintProject(BaseProject): """ projectType = 2 + projectTypeName = "Footprint" def __init__(self, project_draft): # this will create the basis attributes From 3e3eccb9a084cd8616411e9bd7b1eceb946017e3 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Sun, 29 Dec 2019 22:22:05 +0100 Subject: [PATCH 10/51] add projectTypeName as class attribute --- .../footprint/footprint_project.py | 98 ++++++++----------- 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py b/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py index 009ae11e1..b5b048932 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py @@ -7,8 +7,7 @@ from mapswipe_workers.definitions import logger from mapswipe_workers.base.base_project import BaseProject from mapswipe_workers.project_types.footprint import grouping_functions as g -from mapswipe_workers.project_types.footprint.footprint_group \ - import FootprintGroup +from mapswipe_workers.project_types.footprint.footprint_group import FootprintGroup class FootprintProject(BaseProject): @@ -24,42 +23,40 @@ def __init__(self, project_draft): super().__init__(project_draft) # set group size - self.groupSize = project_draft['groupSize'] - self.inputGeometries = project_draft['inputGeometries'] - self.tileServer = self.get_tile_server(project_draft['tileServer']) + self.groupSize = project_draft["groupSize"] + self.inputGeometries = project_draft["inputGeometries"] + self.tileServer = self.get_tile_server(project_draft["tileServer"]) def validate_geometries(self): raw_input_file = ( - f'{DATA_PATH}/' - f'input_geometries/raw_input_{self.projectId}.geojson' - ) + f"{DATA_PATH}/" f"input_geometries/raw_input_{self.projectId}.geojson" + ) valid_input_file = ( - f'{DATA_PATH}/' - f'input_geometries/valid_input_{self.projectId}.geojson' - ) + f"{DATA_PATH}/" f"input_geometries/valid_input_{self.projectId}.geojson" + ) - if not os.path.isdir('{}/input_geometries'.format(DATA_PATH)): - os.mkdir('{}/input_geometries'.format(DATA_PATH)) + if not os.path.isdir("{}/input_geometries".format(DATA_PATH)): + os.mkdir("{}/input_geometries".format(DATA_PATH)) # download file from given url url = self.inputGeometries urllib.request.urlretrieve(url, raw_input_file) logger.info( - f'{self.projectId}' - f' - __init__ - ' - f'downloaded input geometries from url and saved as file: ' - f'{raw_input_file}' - ) + f"{self.projectId}" + f" - __init__ - " + f"downloaded input geometries from url and saved as file: " + f"{raw_input_file}" + ) self.inputGeometries = raw_input_file # open the raw input file and get layer - driver = ogr.GetDriverByName('GeoJSON') + driver = ogr.GetDriverByName("GeoJSON") datasource = driver.Open(raw_input_file, 0) try: layer = datasource.GetLayer() LayerDefn = layer.GetLayerDefn() except AttributeError: - raise CustomError('Value error in input geometries file') + raise CustomError("Value error in input geometries file") # create layer for valid_input_file to store all valid geometries outDriver = ogr.GetDriverByName("GeoJSON") @@ -68,9 +65,8 @@ def validate_geometries(self): outDriver.DeleteDataSource(valid_input_file) outDataSource = outDriver.CreateDataSource(valid_input_file) outLayer = outDataSource.CreateLayer( - "geometries", - geom_type=ogr.wkbMultiPolygon - ) + "geometries", geom_type=ogr.wkbMultiPolygon + ) for i in range(0, LayerDefn.GetFieldCount()): fieldDefn = LayerDefn.GetFieldDefn(i) outLayer.CreateField(fieldDefn) @@ -78,11 +74,9 @@ def validate_geometries(self): # check if raw_input_file layer is empty if layer.GetFeatureCount() < 1: - err = 'empty file. No geometries provided' + err = "empty file. No geometries provided" # TODO: How to user logger and exceptions? - logger.warning( - f'{self.projectId} - check_input_geometry - {err}' - ) + logger.warning(f"{self.projectId} - check_input_geometry - {err}") raise Exception(err) # get geometry as wkt @@ -107,19 +101,19 @@ def validate_geometries(self): if not feat_geom.IsValid(): layer.DeleteFeature(fid) logger.warning( - f'{self.projectId}' - f' - check_input_geometries - ' - f'deleted invalid feature {fid}' - ) + f"{self.projectId}" + f" - check_input_geometries - " + f"deleted invalid feature {fid}" + ) # we accept only POLYGON or MULTIPOLYGON geometries - elif geom_name != 'POLYGON' and geom_name != 'MULTIPOLYGON': + elif geom_name != "POLYGON" and geom_name != "MULTIPOLYGON": layer.DeleteFeature(fid) logger.warning( - f'{self.projectId}' - f' - check_input_geometries - ' - f'deleted non polygon feature {fid}' - ) + f"{self.projectId}" + f" - check_input_geometries - " + f"deleted non polygon feature {fid}" + ) else: # Create output Feature @@ -127,17 +121,16 @@ def validate_geometries(self): # Add field values from input Layer for i in range(0, outLayerDefn.GetFieldCount()): outFeature.SetField( - outLayerDefn.GetFieldDefn(i).GetNameRef(), - feature.GetField(i) - ) + outLayerDefn.GetFieldDefn(i).GetNameRef(), feature.GetField(i) + ) outFeature.SetGeometry(feat_geom) outLayer.CreateFeature(outFeature) outFeature = None # check if layer is empty if layer.GetFeatureCount() < 1: - err = 'no geometries left after checking validity and geometry type.' - logger.warning(f'{self.projectId} - check_input_geometry - {err}') + err = "no geometries left after checking validity and geometry type." + logger.warning(f"{self.projectId} - check_input_geometry - {err}") raise Exception(err) del datasource @@ -147,11 +140,11 @@ def validate_geometries(self): self.validInputGeometries = valid_input_file logger.info( - f'{self.projectId}' - f' - check_input_geometry - ' - f'filtered correct input geometries and created file: ' - f'{valid_input_file}' - ) + f"{self.projectId}" + f" - check_input_geometry - " + f"filtered correct input geometries and created file: " + f"{valid_input_file}" + ) return wkt_geometry def create_groups(self): @@ -159,18 +152,13 @@ def create_groups(self): The function to create groups of footprint geometries """ - raw_groups = g.group_input_geometries( - self.validInputGeometries, - self.groupSize - ) + raw_groups = g.group_input_geometries(self.validInputGeometries, self.groupSize) for group_id, item in raw_groups.items(): group = FootprintGroup(self, group_id) - group.create_tasks(item['feature_ids'], item['feature_geometries']) + group.create_tasks(item["feature_ids"], item["feature_geometries"]) self.groups.append(group) logger.info( - f'{self.projectId} ' - f'- create_groups - ' - f'created groups dictionary' - ) + f"{self.projectId} " f"- create_groups - " f"created groups dictionary" + ) From dcdbf418670eadb23e6be551f12e650b03a70cba Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 14:37:42 +0100 Subject: [PATCH 11/51] Use of global var project_types --- .../mapswipe_workers/mapswipe_workers.py | 40 ++++++------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index b63a1e2c9..a9da795b2 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -1,28 +1,22 @@ +import ast +import json import time + import click import schedule as sched -import json -import ast - -from mapswipe_workers.definitions import CustomError -from mapswipe_workers.definitions import logger -from mapswipe_workers.utils import slack -from mapswipe_workers.utils import sentry from mapswipe_workers import auth -from mapswipe_workers.generate_stats import generate_stats -from mapswipe_workers.firebase_to_postgres import transfer_results -from mapswipe_workers.firebase_to_postgres import update_data -from mapswipe_workers.project_types.build_area.build_area_project import ( - BuildAreaProject, +from mapswipe_workers.definitions import ( + PROJECT_TYPE_CLASSES, + PROJECT_TYPE_NAMES, + CustomError, + logger, ) +from mapswipe_workers.firebase_to_postgres import transfer_results, update_data +from mapswipe_workers.generate_stats import generate_stats from mapswipe_workers.project_types.build_area import build_area_tutorial -from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject -from mapswipe_workers.project_types.change_detection.change_detection_project import ( - ChangeDetectionProject, -) from mapswipe_workers.project_types.change_detection import change_detection_tutorial -from mapswipe_workers.utils import user_management +from mapswipe_workers.utils import sentry, slack, user_management class PythonLiteralOption(click.Option): @@ -346,14 +340,6 @@ def _run(): def _run_create_projects(project_draft_ids=None): - project_types = { - # Make sure to import all project types here - 1: BuildAreaProject, - 2: FootprintProject, - 3: ChangeDetectionProject, - } - project_type_names = {1: "Build Area", 2: "Footprint", 3: "Change Detection"} - fb_db = auth.firebaseDB() ref = fb_db.reference("v2/projectDrafts/") project_drafts = ref.get() @@ -381,7 +367,7 @@ def _run_create_projects(project_draft_ids=None): project_type = project_draft.get("projectType", 1) try: # TODO: Document properly - project = project_types[project_type](project_draft) + project = PROJECT_TYPE_CLASSES[project_type](project_draft) project.geometry = project.validate_geometries() project.create_groups() project.calc_required_results() @@ -392,7 +378,7 @@ def _run_create_projects(project_draft_ids=None): f"### PROJECT CREATION SUCCESSFUL ###{newline}" f"Project Name: {project.name}{newline}" f"Project Id: {project.projectId}{newline}" - f"Project Type: {project_type_names[project_type]}" + f"Project Type: {PROJECT_TYPE_NAMES[project_type]}" f"{newline}" f"Make sure to activate the project " f"using the manager dashboard." From 56620104f51575881de843184ed83b48fbd824ea Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 15:01:18 +0100 Subject: [PATCH 12/51] update requirements --- mapswipe_workers/requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mapswipe_workers/requirements.txt b/mapswipe_workers/requirements.txt index db9740af1..7a71a3ec1 100644 --- a/mapswipe_workers/requirements.txt +++ b/mapswipe_workers/requirements.txt @@ -1,12 +1,12 @@ black==19.10b0 click==7.0 -firebase-admin==2.17.0 +firebase-admin==3.2.1 flake8==3.7.9 mapswipe-workers==3.0 -pandas==0.25.2 -pre-commit==1.20.0 -psycopg2-binary==2.8.3 -python-dateutil==2.8.0 +pandas==0.25.3 +pre-commit==1.21.0 +psycopg2-binary==2.8.4 +python-dateutil==2.8.1 schedule==0.6.0 -sentry-sdk==0.12.3 -slackclient==2.1.0 +sentry-sdk==0.14.0 +slackclient==2.5.0 From 7dce91690f81588e1ad06c4518809239067aa1d2 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 17:23:13 +0100 Subject: [PATCH 13/51] refactor code to only initialize sentry and slack. Sending messages should be done by utilizing standart methods provide by those modules. --- .../mapswipe_workers/utils/init_sentry.py | 12 ++ .../mapswipe_workers/utils/init_slack.py | 12 ++ .../mapswipe_workers/utils/sentry.py | 25 ----- .../mapswipe_workers/utils/slack.py | 103 ------------------ 4 files changed, 24 insertions(+), 128 deletions(-) create mode 100644 mapswipe_workers/mapswipe_workers/utils/init_sentry.py create mode 100644 mapswipe_workers/mapswipe_workers/utils/init_slack.py delete mode 100644 mapswipe_workers/mapswipe_workers/utils/sentry.py delete mode 100644 mapswipe_workers/mapswipe_workers/utils/slack.py diff --git a/mapswipe_workers/mapswipe_workers/utils/init_sentry.py b/mapswipe_workers/mapswipe_workers/utils/init_sentry.py new file mode 100644 index 000000000..031904144 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/utils/init_sentry.py @@ -0,0 +1,12 @@ +"""Initialize sentry client with values provided by the config file""" + +import json +import sentry_sdk + +from mapswipe_workers.definitions import CONFIG_PATH + + +with open(CONFIG_PATH) as config_file: + config = json.load(config_file) + sentry_url = config["sentry"]["dsn"] + sentry_sdk.init(sentry_url) diff --git a/mapswipe_workers/mapswipe_workers/utils/init_slack.py b/mapswipe_workers/mapswipe_workers/utils/init_slack.py new file mode 100644 index 000000000..f4b504f90 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/utils/init_slack.py @@ -0,0 +1,12 @@ +"""Initialize slack client with values provided by the config file""" + +import json +import slack + +from mapswipe_workers.definitions import CONFIG_PATH + + +with open(CONFIG_PATH) as config_file: + config = json.load(config_file) + slack_channel = config["slack"]["channel"] + slack_client = slack.WebClient(token=config["slack"]["token"]) diff --git a/mapswipe_workers/mapswipe_workers/utils/sentry.py b/mapswipe_workers/mapswipe_workers/utils/sentry.py deleted file mode 100644 index c4568bf43..000000000 --- a/mapswipe_workers/mapswipe_workers/utils/sentry.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -import sentry_sdk -from sentry_sdk import capture_exception -from sentry_sdk import capture_message -from mapswipe_workers.definitions import CONFIG_PATH - -def init_sentry(): - - try: - with open(CONFIG_PATH) as json_data_file: - data = json.load(json_data_file) - sentry_url = data['sentry']['dsn'] - sentry_sdk.init(sentry_url) - return True - except: - print('no sentry config provided') - return None - - -def capture_exception_sentry(error): - capture_exception(error) - - -def capture_message_sentry(message): - capture_message(message) \ No newline at end of file diff --git a/mapswipe_workers/mapswipe_workers/utils/slack.py b/mapswipe_workers/mapswipe_workers/utils/slack.py deleted file mode 100644 index ca27feb31..000000000 --- a/mapswipe_workers/mapswipe_workers/utils/slack.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/python3 -# -*- coding: UTF-8 -*- -# Author: M. Reinmuth, B. Herfort - -import sys -import traceback -import json -import slack -from mapswipe_workers.definitions import CONFIG_PATH - - -def get_slack_client(): - """ - The function to init the slack client from information provided in config file - - Returns - ------- - sc : slack.WebClient - channel: str - name of slack channel - username : str - user name for the bot in slack - """ - - try: - with open(CONFIG_PATH) as json_data_file: - data = json.load(json_data_file) - slack_token = data['slack']['token'] - channel = data['slack']['channel'] - username = data['slack']['username'] - sc = slack.WebClient(token=slack_token) - return sc, channel, username - except: - print('no slack token provided') - return None - - -def send_slack_message(msg): - """ - The function to send a message to a slack channel - - Parameters - ---------- - msg : str - the message to send as string - - Returns - ------- - bool - True if successful, False otherwise - """ - - if not get_slack_client(): - return False - else: - sc, channel, username = get_slack_client() - response = sc.chat_postMessage( - as_user=True, - channel=channel, - text=msg, - username=username, - ) - assert response["ok"] - return True - - -def _get_error_message_details(error): - """ - The function to nicely extract error text and traceback." - Parameters - ---------- - error : Exception - the python exception which caused the error - Returns - ------- - error_msg_string : str - """ - - type_, value_, traceback_ = sys.exc_info() - error_msg = traceback.format_exception(type_, value_, traceback_) - error_msg_string = '' - for part in error_msg: - error_msg_string += part + '\n' - return error_msg_string - - -def send_error(error): - """ - The function to send an error message to Slack - Parameters - ---------- - error : Exception - the python exception which caused the error - Returns - ------- - bool - True if successful, false otherwise. - """ - - error_msg = _get_error_message_details(error) - head = 'python-mapswipe-workers: error occured' - send_slack_message(head + '\n' + error_msg) - return True \ No newline at end of file From c25f2423a79c5cadbac02696483d6be7616a1f1d Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 17:24:42 +0100 Subject: [PATCH 14/51] Remove user attribute from slack config. This is only usefull if messages should be sent to a specific user. --- mapswipe_workers/config/example-configuration.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mapswipe_workers/config/example-configuration.json b/mapswipe_workers/config/example-configuration.json index 68fa41c73..b979574fe 100644 --- a/mapswipe_workers/config/example-configuration.json +++ b/mapswipe_workers/config/example-configuration.json @@ -26,8 +26,7 @@ }, "slack": { "token": "your_slack_token", - "channel": "your_slack_channel", - "username": "your_slack_username" + "channel": "your_slack_channel" }, "sentry": { "dsn": "your_sentry_dsn_value" From 182cb0d2bf9cb46ba99aea1afa70532363009f68 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 17:33:31 +0100 Subject: [PATCH 15/51] Rename attributes accourding to conventions. --- mapswipe_workers/mapswipe_workers/definitions.py | 6 +++--- .../project_types/build_area/build_area_project.py | 4 ++-- .../change_detection/change_detection_project.py | 4 ++-- .../project_types/footprint/footprint_project.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index 41c7002c7..b9b5be5f8 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -29,9 +29,9 @@ } PROJECT_TYPE_NAMES = { - 1: BuildAreaProject.name, - 2: FootprintProject.name, - 3: ChangeDetectionProject.name, + 1: BuildAreaProject.project_type_name, + 2: FootprintProject.project_type_name, + 3: ChangeDetectionProject.project_type_name, } logging.config.fileConfig(fname=LOGGING_CONFIG_PATH, disable_existing_loggers=True) diff --git a/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py b/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py index 2e54dd8a0..f79a546e6 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py @@ -16,8 +16,8 @@ class BuildAreaProject(BaseProject): The subclass for an import of the type Footprint """ - projectType = 1 - projectTypeName = "Build Area" + project_type = 1 + project_type_name = "Build Area" def __init__(self, project_draft): # this will create the basis attributes diff --git a/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py b/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py index c8b745fac..a97aec70f 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py @@ -18,8 +18,8 @@ class ChangeDetectionProject(BaseProject): The subclass for an import of the type Footprint """ - projectType = 1 - projectTypeName = "Change Detection" + project_type = 3 + project_type_name = "Change Detection" def __init__(self, project_draft): # this will create the basis attributes diff --git a/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py b/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py index b5b048932..579079036 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py @@ -15,8 +15,8 @@ class FootprintProject(BaseProject): The subclass for an import of the type Footprint """ - projectType = 2 - projectTypeName = "Footprint" + project_type = 2 + project_type_name = "Footprint" def __init__(self, project_draft): # this will create the basis attributes From 619923b3fb065bce658725a10819d83f31616b03 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 17:51:36 +0100 Subject: [PATCH 16/51] Remove not needed attribute 'archive' --- postgres/initdb.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/postgres/initdb.sql b/postgres/initdb.sql index 991f623b6..107114159 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -2,7 +2,6 @@ CREATE EXTENSION postgis; CREATE TABLE IF NOT EXISTS projects ( - archive boolean, created timestamp, created_by varchar, geom geometry(MULTIPOLYGON, 4326), @@ -22,7 +21,7 @@ CREATE TABLE IF NOT EXISTS projects ( PRIMARY KEY (project_id) ); -CREATE TABLE IF NOT EXISTS groups ( +CREATE TABLE IF NOT EXISTS GROUPS ( project_id varchar, group_id varchar, number_of_tasks int, From 66d10d06ad756448526a670a9d5c77afdcd07d34 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 17:52:23 +0100 Subject: [PATCH 17/51] Change status instead of dedicated archived attribute --- .../firebase_to_postgres/archive_project.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py index b4c6ec3e7..dcba3b394 100644 --- a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py +++ b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py @@ -19,8 +19,12 @@ def archive_project(project_id: str) -> None: logger.info("Archive project with the id {0}".format(project_id)) delete_project_from_firebase(project_id) + fb_db = auth.firebaseDB() + ref = fb_db.reference("v2/projects/{0}/status".format(project_id)) + ref.set({"archived"}) + pg_db = auth.postgresDB() - sql_query = "UPDATE projects SET archived = true WHERE project_id = {0}".format( + sql_query = "UPDATE projects SET status = 'archived' WHERE project_id = {0}".format( project_id ) pg_db.query(sql_query) From 58a1e4c14512d15001eb8dcd6b4e380f8a056f36 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 17:54:32 +0100 Subject: [PATCH 18/51] Test status instead of dedicated archived attribute --- mapswipe_workers/tests/test_archive_project.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/tests/test_archive_project.py b/mapswipe_workers/tests/test_archive_project.py index a8b350432..daf565e2b 100644 --- a/mapswipe_workers/tests/test_archive_project.py +++ b/mapswipe_workers/tests/test_archive_project.py @@ -27,6 +27,9 @@ def test_firebase_changes(self): """Test if groups, tasks and results are deleted from Firebase.""" fb_db = auth.firebaseDB() + ref = fb_db.reference("v2/projects/{0}/status".format(self.project_id)) + self.assertEqual(ref.get(), "archived") + ref = fb_db.reference("v2/groups/{0}".format(self.project_id)) self.assertFalse(ref.get()) @@ -39,7 +42,7 @@ def test_firebase_changes(self): def test_postgres_changes(self): """Test if postgres project is archived.""" pg_db = auth.postgresDB() - sql_query = "SELECT archived FROM projects WHERE project_id = {}".format( + sql_query = "SELECT status FROM projects WHERE project_id = {}".format( self.project_id ) result = pg_db.retr_query(sql_query) From 462b5cb08c511fcb22c11b759012a96218528508 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 18:01:24 +0100 Subject: [PATCH 19/51] correct missspelled entity 'group' --- postgres/initdb.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/postgres/initdb.sql b/postgres/initdb.sql index 107114159..7ab77ae09 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS projects ( PRIMARY KEY (project_id) ); -CREATE TABLE IF NOT EXISTS GROUPS ( +CREATE TABLE IF NOT EXISTS groups ( project_id varchar, group_id varchar, number_of_tasks int, @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), - FOREIGN KEY (project_id, group_id) REFERENCES GROUPS (project_id, group_id) + FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id) ); CREATE INDEX IF NOT EXISTS tasks_task_id ON public.tasks @@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS results ( result int, PRIMARY KEY (project_id, group_id, task_id, user_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), - FOREIGN KEY (project_id, group_id) REFERENCES GROUPS (project_id, group_id), + FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id), FOREIGN KEY (project_id, group_id, task_id) REFERENCES tasks (project_id, group_id, task_id), FOREIGN KEY (user_id) REFERENCES users (user_id) ); From c7130fe15d38be613d53653591e76b7e7bcbbe87 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 18:15:50 +0100 Subject: [PATCH 20/51] Revert "correct missspelled entity 'group'" This reverts commit 462b5cb08c511fcb22c11b759012a96218528508. --- postgres/initdb.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/postgres/initdb.sql b/postgres/initdb.sql index 7ab77ae09..107114159 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS projects ( PRIMARY KEY (project_id) ); -CREATE TABLE IF NOT EXISTS groups ( +CREATE TABLE IF NOT EXISTS GROUPS ( project_id varchar, group_id varchar, number_of_tasks int, @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), - FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id) + FOREIGN KEY (project_id, group_id) REFERENCES GROUPS (project_id, group_id) ); CREATE INDEX IF NOT EXISTS tasks_task_id ON public.tasks @@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS results ( result int, PRIMARY KEY (project_id, group_id, task_id, user_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), - FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id), + FOREIGN KEY (project_id, group_id) REFERENCES GROUPS (project_id, group_id), FOREIGN KEY (project_id, group_id, task_id) REFERENCES tasks (project_id, group_id, task_id), FOREIGN KEY (user_id) REFERENCES users (user_id) ); From e84bf8445e7023727ab526ffbe0c98bf769105c4 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 18:18:48 +0100 Subject: [PATCH 21/51] Revert "Test status instead of dedicated archived attribute" This reverts commit 58a1e4c14512d15001eb8dcd6b4e380f8a056f36. --- mapswipe_workers/tests/test_archive_project.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mapswipe_workers/tests/test_archive_project.py b/mapswipe_workers/tests/test_archive_project.py index daf565e2b..a8b350432 100644 --- a/mapswipe_workers/tests/test_archive_project.py +++ b/mapswipe_workers/tests/test_archive_project.py @@ -27,9 +27,6 @@ def test_firebase_changes(self): """Test if groups, tasks and results are deleted from Firebase.""" fb_db = auth.firebaseDB() - ref = fb_db.reference("v2/projects/{0}/status".format(self.project_id)) - self.assertEqual(ref.get(), "archived") - ref = fb_db.reference("v2/groups/{0}".format(self.project_id)) self.assertFalse(ref.get()) @@ -42,7 +39,7 @@ def test_firebase_changes(self): def test_postgres_changes(self): """Test if postgres project is archived.""" pg_db = auth.postgresDB() - sql_query = "SELECT status FROM projects WHERE project_id = {}".format( + sql_query = "SELECT archived FROM projects WHERE project_id = {}".format( self.project_id ) result = pg_db.retr_query(sql_query) From 6a87ffdf6e71f59af97837f294054ddb5af0633f Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Mon, 13 Jan 2020 18:22:00 +0100 Subject: [PATCH 22/51] Correct missspelled entity 'GROUPS' to groups --- postgres/initdb.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/postgres/initdb.sql b/postgres/initdb.sql index 107114159..7ab77ae09 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS projects ( PRIMARY KEY (project_id) ); -CREATE TABLE IF NOT EXISTS GROUPS ( +CREATE TABLE IF NOT EXISTS groups ( project_id varchar, group_id varchar, number_of_tasks int, @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), - FOREIGN KEY (project_id, group_id) REFERENCES GROUPS (project_id, group_id) + FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id) ); CREATE INDEX IF NOT EXISTS tasks_task_id ON public.tasks @@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS results ( result int, PRIMARY KEY (project_id, group_id, task_id, user_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), - FOREIGN KEY (project_id, group_id) REFERENCES GROUPS (project_id, group_id), + FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id), FOREIGN KEY (project_id, group_id, task_id) REFERENCES tasks (project_id, group_id, task_id), FOREIGN KEY (user_id) REFERENCES users (user_id) ); From c1514171679ef64b6a72fbf050a7496dc25422e2 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 14 Jan 2020 10:44:19 +0100 Subject: [PATCH 23/51] integrate delete_project() functionality in main archive function and improve docstrings --- .../firebase_to_postgres/archive_project.py | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py index dcba3b394..90c4198cc 100644 --- a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py +++ b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py @@ -1,8 +1,5 @@ """ Archive a project. - -Delete groups, tasks and results of a project in Firebase. -Keep all informations in Postgres. """ from mapswipe_workers import auth @@ -13,11 +10,18 @@ def archive_project(project_id: str) -> None: """ Archive a project. - Call a function which deletes groups, tasks and results in Firebase. - Set archive = true for project in Postgres. + Deletes groups, tasks and results from Firebase. + Set status = archived for project in Firebase and Postgres. """ logger.info("Archive project with the id {0}".format(project_id)) - delete_project_from_firebase(project_id) + logger.info( + "Delete results, groups and tasks of project with the id {0}".format(project_id) + ) + + fb_db = auth.firebaseDB() + fb_db.reference("v2/results/{0}".format(project_id)).set({}) + fb_db.reference("v2/groups/{0}".format(project_id)).set({}) + fb_db.reference("v2/tasks/{0}".format(project_id)).set({}) fb_db = auth.firebaseDB() ref = fb_db.reference("v2/projects/{0}/status".format(project_id)) @@ -28,14 +32,3 @@ def archive_project(project_id: str) -> None: project_id ) pg_db.query(sql_query) - - -def delete_project_from_firebase(project_id: str) -> None: - """Delete results, groups and tasks of given project in Firebase.""" - logger.info( - "Delete results, groups and tasks of project with the id {0}".format(project_id) - ) - fb_db = auth.firebaseDB() - fb_db.reference("v2/results/{0}".format(project_id)).set({}) - fb_db.reference("v2/groups/{0}".format(project_id)).set({}) - fb_db.reference("v2/tasks/{0}".format(project_id)).set({}) From 62edef7af7e413b699e11a6eacca40054c39feee Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 14 Jan 2020 10:45:01 +0100 Subject: [PATCH 24/51] Support lists of project ids for archive argument --- .../mapswipe_workers/mapswipe_workers.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index 73f7e6c44..b72c74a1a 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -298,15 +298,25 @@ def run_create_tutorial(input_file): @click.option( "--project-id", "-i", + help=("Archive project with giving project id"), + type=str, +) +@click.option( + "--project-ids", + cls=PythonLiteralOption, help=( - "Archive project in Postgres. Delete groups, tasks and results from Firebase." + f"Archive multiple projects. " + f"Provide project id strings as a list: " + f"""["project_a", "project_b"]""" ), - type=str, ) -def run_archive_project(project_id): - update_data.update_project_data([project_id]) - transfer_results.transfer_results([project_id]) - archive_project.archive_project(project_id) +def run_archive_project(project_id, project_ids): + """Archive projects in Postgres. Delete groups, tasks and results from Firebase.""" + if not project_ids: + project_ids = [project_id] + update_data.update_project_data(project_ids) + transfer_results.transfer_results(project_ids) + archive_project.archive_project(project_ids) @click.command("run") From e6bebfecf1ccc07194f1ded33785945a53505a25 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 14 Jan 2020 10:52:16 +0100 Subject: [PATCH 25/51] Support for a list of project ids as argument for archive function. --- .../firebase_to_postgres/archive_project.py | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py index 90c4198cc..c72465240 100644 --- a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py +++ b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py @@ -6,29 +6,33 @@ from mapswipe_workers.definitions import logger -def archive_project(project_id: str) -> None: +def archive_project(project_ids: list) -> None: """ Archive a project. Deletes groups, tasks and results from Firebase. Set status = archived for project in Firebase and Postgres. """ - logger.info("Archive project with the id {0}".format(project_id)) - logger.info( - "Delete results, groups and tasks of project with the id {0}".format(project_id) - ) + for project_id in project_ids: + logger.info("Archive project with the id {0}".format(project_id)) + logger.info( + "Delete results, groups and tasks of project with the id {0}".format( + project_id + ) + ) - fb_db = auth.firebaseDB() - fb_db.reference("v2/results/{0}".format(project_id)).set({}) - fb_db.reference("v2/groups/{0}".format(project_id)).set({}) - fb_db.reference("v2/tasks/{0}".format(project_id)).set({}) + fb_db = auth.firebaseDB() + fb_db.reference("v2/results/{0}".format(project_id)).set({}) + fb_db.reference("v2/groups/{0}".format(project_id)).set({}) + fb_db.reference("v2/tasks/{0}".format(project_id)).set({}) - fb_db = auth.firebaseDB() - ref = fb_db.reference("v2/projects/{0}/status".format(project_id)) - ref.set({"archived"}) + fb_db = auth.firebaseDB() + ref = fb_db.reference("v2/projects/{0}/status".format(project_id)) + ref.set({"archived"}) - pg_db = auth.postgresDB() - sql_query = "UPDATE projects SET status = 'archived' WHERE project_id = {0}".format( - project_id - ) - pg_db.query(sql_query) + pg_db = auth.postgresDB() + sql_query = ( + "UPDATE projects SET status = 'archived' " + + "WHERE project_id = {0}".format(project_id) + ) + pg_db.query(sql_query) From 35240f497d9c1e49b66bc4a3ca6c63ba1efa26f4 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 14 Jan 2020 13:57:24 +0100 Subject: [PATCH 26/51] --schedule option defaults now to 10 minutes. --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 0df0e929a..f476de578 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -53,7 +53,7 @@ services: context: mapswipe_workers/ depends_on: - postgres - command: mapswipe_workers --verbose run --schedule m + command: mapswipe_workers --verbose run --schedule volumes: - ./api-data:/var/lib/mapswipe_workers/api-data/:rw - ./api-data/agg_results:/var/lib/mapswipe_workers/api-data/agg_results:rw From 5a2eb16e1efce19b1cf2f7cce76714eb912db7a4 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 14 Jan 2020 14:01:41 +0100 Subject: [PATCH 27/51] Create fb_db (Firbase sdk) when needed. Remove passing it to functions. --- mapswipe_workers/mapswipe_workers/base/base_project.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/base/base_project.py b/mapswipe_workers/mapswipe_workers/base/base_project.py index 53bb5da9f..979b848c3 100644 --- a/mapswipe_workers/mapswipe_workers/base/base_project.py +++ b/mapswipe_workers/mapswipe_workers/base/base_project.py @@ -79,7 +79,7 @@ def __init__(self, project_draft): # def resultRequiredCounter(self): # return self.resultRequiredCounter - def save_project(self, fb_db): + def save_project(self): """ Creates a projects with groups and tasks and saves it in firebase and postgres @@ -131,7 +131,6 @@ def save_project(self, fb_db): ) try: self.save_to_firebase( - fb_db, project, groups, groupsOfTasks, @@ -161,7 +160,7 @@ def save_project(self, fb_db): ) raise CustomError(e) - def save_to_firebase(self, fb_db, project, groups, groupsOfTasks): + def save_to_firebase(self, project, groups, groupsOfTasks): # remove wkt geometry attribute of projects and tasks project.pop('geometry', None) @@ -170,6 +169,7 @@ def save_to_firebase(self, fb_db, project, groups, groupsOfTasks): groupsOfTasks[group_id][i].pop('geometry', None) + fb_db = auth.firebaseDB() ref = fb_db.reference('') # save project ref.update({ From f264ee5f7c687267481e2e7920e26a441948643b Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 14 Jan 2020 14:02:56 +0100 Subject: [PATCH 28/51] add ms as shorthand for mapswipe_workers command --- mapswipe_workers/setup.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/mapswipe_workers/setup.py b/mapswipe_workers/setup.py index 66d2a123c..2203dd150 100644 --- a/mapswipe_workers/setup.py +++ b/mapswipe_workers/setup.py @@ -1,22 +1,23 @@ from setuptools import setup, find_packages -console_scripts=''' +console_scripts = """ [console_scripts] mapswipe_workers=mapswipe_workers.mapswipe_workers:cli - ''' + ms=mapswipe_workers.mapswipe_workers:cli + """ -with open('requirements.txt') as f: +with open("requirements.txt") as f: requirements = f.read().splitlines() setup( - name='mapswipe-workers', - version='3.0', - description='Install script for the mapswipe Python workers.', - author='B. Herfort, M. Schaub, M. Reinmuth', - author_email='', - url='www.mapswipe.org', - packages=find_packages(exclude=('tests', 'docs')), + name="mapswipe-workers", + version="3.0", + description="Install script for the MapSwipe-Python-Workers.", + author="B. Herfort, M. Schaub, M. Reinmuth", + author_email="", + url="www.mapswipe.org", + packages=find_packages(exclude=("tests", "docs")), install_requires=requirements, - entry_points=console_scripts + entry_points=console_scripts, ) From c9a25e0eca249060d36966a25920b900650e8e29 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 14 Jan 2020 14:34:22 +0100 Subject: [PATCH 29/51] Use scheduling only for run(). Simplified code. Run Black and Flake8. --- .../mapswipe_workers/mapswipe_workers.py | 450 +++++------------- 1 file changed, 110 insertions(+), 340 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index a9da795b2..8f9652585 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -1,10 +1,11 @@ +"""Command Line Interface for MapSwipe Workers.""" + import ast import json import time import click import schedule as sched - from mapswipe_workers import auth from mapswipe_workers.definitions import ( PROJECT_TYPE_CLASSES, @@ -29,211 +30,105 @@ def type_cast_value(self, ctx, value): @click.group() -@click.option( - "--verbose", "-v", is_flag=True, -) +@click.option("--verbose", "-v", is_flag=True, help="Enable logging.") @click.version_option() def cli(verbose): + """Enable logging.""" if not verbose: logger.disabled = True -@click.command("create-projects") -@click.option( - "--schedule", - "-s", - default=None, - help=( - f"Will create projects every " - f"10 minutes (m), every hour (h) or every day (d). " - ), - type=click.Choice(["m", "h", "d"]), -) -def run_create_projects(schedule): - sentry.init_sentry() - try: - if schedule: - if schedule == "m": - sched.every(10).minutes.do(_run_create_projects).run() - while True: - sched.run_pending() - time.sleep(1) - elif schedule == "h": - sched.every().hour.do(_run_create_projects).run() - while True: - sched.run_pending() - time.sleep(1) - elif schedule == "d": - sched.every().day.do(_run_create_projects).run() - while True: - sched.run_pending() - time.sleep(1) - else: - click.echo( - f"{schedule} is not a valid input " - f"for the schedule argument. " - f"Use m for every 10 minutes, " - f"h for every hour and d for every day." - ) - else: - _run_create_projects() - except Exception as e: - slack.send_error(e) - sentry.capture_exception_sentry(e) - logger.exception(e) +@cli.command("create-projects") +def run_create_projects(): + """ + Create projects from submitted project drafts. + Get project drafts from Firebase. + Create projects with groups and tasks. + Save created projects, groups and tasks to Firebase and Postgres. + """ + fb_db = auth.firebaseDB() + ref = fb_db.reference("v2/projectDrafts/") + project_drafts = ref.get() -@click.command("firebase-to-postgres") -@click.option( - "--schedule", - "-s", - default=None, - help=( - f"Will update and transfer relevant data (i.a. users and results) " - f"from Firebase into Postgres " - f"every 10 minutes (m), every hour (h) or every day (d). " - ), - type=click.Choice(["m", "h", "d"]), -) -def run_firebase_to_postgres(schedule): - sentry.init_sentry() - try: - if schedule: - if schedule == "m": - sched.every(10).minutes.do(_run_firebase_to_postgres).run() - while True: - sched.run_pending() - time.sleep(1) - elif schedule == "h": - sched.every().hour.do(_run_firebase_to_postgres).run() - while True: - sched.run_pending() - time.sleep(1) - elif schedule == "d": - sched.every().day.do(_run_firebase_to_postgres).run() - while True: - sched.run_pending() - time.sleep(1) - else: - click.echo( - f"{schedule} is not a valid input " - f"for the schedule argument. " - f"Use m for every 10 minutes, " - f"h for every hour and d for every day." - ) - else: - _run_firebase_to_postgres() - except Exception as e: - slack.send_error(e) - sentry.capture_exception_sentry(e) - logger.exception(e) + if project_drafts is None: + return None + for project_draft_id, project_draft in project_drafts.items(): + project_draft["projectDraftId"] = project_draft_id + project_type = project_draft["projectType"] + try: + # Create a project object using appropriate class (project type). + project = PROJECT_TYPE_CLASSES[project_type](project_draft) + project.geometry = project.validate_geometries() + project.create_groups() + project.calc_required_results() + # Save project and its groups and tasks to Firebase and Postgres. + project.save_project() + newline = "\n" + message = ( + f"### PROJECT CREATION SUCCESSFUL ###{newline}" + f"Project Name: {project.name}{newline}" + f"Project Id: {project.projectId}{newline}" + f"Project Type: {PROJECT_TYPE_NAMES[project_type]}" + f"{newline}" + f"Make sure to activate the project " + f"using the manager dashboard." + f"{newline}" + f"Happy Swiping. :)" + ) + slack.send_slack_message(message) + logger.info(message) + except CustomError: + ref = fb_db.reference(f"v2/projectDrafts/{project_draft_id}") + ref.set({}) + newline = "\n" + message = ( + f"### PROJECT CREATION FAILED ###{newline}" + f'Project Name: {project_draft["name"]}{newline}' + f"Project Id: {project_draft_id}{newline}" + f"{newline}" + f"Project draft is deleted.{newline}" + f"Please check what went wrong." + ) + slack.send_slack_message(message) + slack.send_error(CustomError) + logger.exception(f"{project_draft_id} " f"- project creation failed") + sentry.capture_exception() + continue + + +@cli.command("firebase-to-postgres") +def run_firebase_to_postgres() -> list: + """Update users and transfer results from Firebase to Postgres.""" + update_data.update_user_data() + update_data.update_project_data() + project_ids = transfer_results.transfer_results() + return project_ids -@click.command("generate-stats") -@click.option( - "--schedule", - "-s", - default=None, - help=( - f"Generate stats every " f"10 minutes (m), every hour (h) or every day (d). " - ), - type=click.Choice(["m", "h", "d"]), -) + +@cli.command("generate-stats") @click.option( - "--project_id_list", + "--project_ids", cls=PythonLiteralOption, default="[]", help=( - f"provide project id strings as a list " - f"stats will be generated only for this" - f"""use it like '["project_a", "project_b"]' """ + "Project ids for which to generate stats as a list of strings: " + + """["project_a", "project_b"]""" ), ) -def run_generate_stats(schedule, project_id_list): - sentry.init_sentry() - try: - if schedule: - if schedule == "m": - sched.every(10).minutes.do( - _run_generate_stats, project_id_list=project_id_list - ).run() - while True: - sched.run_pending() - time.sleep(1) - elif schedule == "h": - sched.every().hour.do( - _run_generate_stats, project_id_list=project_id_list - ).run() - while True: - sched.run_pending() - time.sleep(1) - elif schedule == "d": - sched.every().day.do( - _run_generate_stats, project_id_list=project_id_list - ).run() - while True: - sched.run_pending() - time.sleep(1) - else: - click.echo( - f"{schedule} is not a valid input " - f"for the schedule argument. " - f"Use m for every 10 minutes, " - f"h for every hour and d for every day." - ) - else: - _run_generate_stats(project_id_list) - except Exception as e: - slack.send_error(e) - sentry.capture_exception_sentry(e) - logger.exception(e) +def run_generate_stats(project_ids: list) -> None: + """Generate statistics for given project ids.""" + generate_stats.generate_stats(project_ids) -@click.command("generate-stats-all-projects") -@click.option( - "--schedule", - "-s", - default=None, - help=( - f"Generate stats every " f"10 minutes (m), every hour (h) or every day (d). " - ), - type=click.Choice(["m", "h", "d"]), -) -def run_generate_stats_all_projects(schedule): - sentry.init_sentry() - try: - if schedule: - if schedule == "m": - sched.every(10).minutes.do(_run_generate_stats_all_projects).run() - while True: - sched.run_pending() - time.sleep(1) - elif schedule == "h": - sched.every().hour.do(_run_generate_stats_all_projects).run() - while True: - sched.run_pending() - time.sleep(1) - elif schedule == "d": - sched.every().day.do(_run_generate_stats_all_projects).run() - while True: - sched.run_pending() - time.sleep(1) - else: - click.echo( - f"{schedule} is not a valid input " - f"for the schedule argument. " - f"Use m for every 10 minutes, " - f"h for every hour and d for every day." - ) - else: - _run_generate_stats_all_projects() - except Exception as e: - slack.send_error(e) - sentry.capture_exception_sentry(e) - logger.exception(e) +@cli.command("generate-stats-all-projects") +def run_generate_stats_all_projects() -> None: + """Generate statistics for all projects.""" + generate_stats.generate_stats_all_projects() -@click.command("user-management") +@cli.command("user-management") @click.option( "--email", help=(f"The email of the MapSwipe user."), required=True, type=str ) @@ -246,11 +141,13 @@ def run_generate_stats_all_projects(schedule): ), type=bool, ) -def run_user_management(email, manager): - sentry.init_sentry() +def run_user_management(email, manager) -> None: try: - if email: - _run_user_management(email, manager) + if email and manager: + if manager: + user_management.set_project_manager_rights(email) + else: + user_management.remove_project_manager_rights(email) else: click.echo(f"Please provide all required input arguments.") except Exception as e: @@ -259,15 +156,14 @@ def run_user_management(email, manager): logger.exception(e) -@click.command("create-tutorial") +@cli.command("create-tutorial") @click.option( "--input_file", help=(f"The json file with your tutorial information."), required=True, type=str, ) -def run_create_tutorial(input_file): - sentry.init_sentry() +def run_create_tutorial(input_file) -> None: try: logger.info(f"will generate tutorial based on {input_file}") with open(input_file) as json_file: @@ -288,153 +184,27 @@ def run_create_tutorial(input_file): logger.exception(e) -@click.command("run") +@cli.command("run") @click.option( - "--schedule", - "-s", - default=None, - help=( - f"Will run Mapswipe Workers every " - f"10 minutes (m), every hour (h) or every day (d). " - ), - type=click.Choice(["m", "h", "d"]), + "--schedule", is_flag=True, help=("Schedule jobs to run every 10 minutes.") ) -def run(schedule): - sentry.init_sentry() - try: - if schedule: - if schedule == "m": - sched.every(10).minutes.do(_run).run() - while True: - sched.run_pending() - time.sleep(1) - elif schedule == "h": - sched.every().hour.do(_run).run() - while True: - sched.run_pending() - time.sleep(1) - elif schedule == "d": - sched.every().day.do(_run).run() - while True: - sched.run_pending() - time.sleep(1) - else: - click.echo( - f"{schedule} is not a valid input " - f"for the schedule argument. " - f"Use m for every 10 minutes, " - f"h for every hour and d for every day." - ) - else: - _run() - except Exception as e: - slack.send_error(e) - sentry.capture_exception_sentry(e) - logger.exception(e) - - -def _run(): - _run_create_projects() - project_id_list = _run_firebase_to_postgres() - _run_generate_stats(project_id_list) - - -def _run_create_projects(project_draft_ids=None): - fb_db = auth.firebaseDB() - ref = fb_db.reference("v2/projectDrafts/") - project_drafts = ref.get() - - if project_drafts is None: - del fb_db - return None - +def run(schedule: bool) -> None: + """ + Run all commands. + + Run --create-projects, --firebase-to-postgres and --generate_stats_all_projects. + If schedule option is set above commands will be run every 10 minutes sequentially. + """ + + def _run(): + run_create_projects() + project_ids = run_firebase_to_postgres() + run_generate_stats(project_ids) + + if schedule: + sched.every(10).minutes.do(_run) + while True: + sched.run_pending() + time.sleep(1) else: - created_project_ids = list() - - for project_draft_id, project_draft in project_drafts.items(): - project_draft["projectDraftId"] = project_draft_id - - # filter out project which are not in project_ids list - # this is only done if a list is provided - - if project_draft_ids: - if project_draft_id not in project_draft_ids: - # pass projects that are not in provided list - continue - - # Early projects have no projectType attribute. - # If so it defaults to 1 - project_type = project_draft.get("projectType", 1) - try: - # TODO: Document properly - project = PROJECT_TYPE_CLASSES[project_type](project_draft) - project.geometry = project.validate_geometries() - project.create_groups() - project.calc_required_results() - if project.save_project(fb_db): - created_project_ids.append(project.projectId) - newline = "\n" - message = ( - f"### PROJECT CREATION SUCCESSFUL ###{newline}" - f"Project Name: {project.name}{newline}" - f"Project Id: {project.projectId}{newline}" - f"Project Type: {PROJECT_TYPE_NAMES[project_type]}" - f"{newline}" - f"Make sure to activate the project " - f"using the manager dashboard." - f"{newline}" - f"Happy Swiping. :)" - ) - slack.send_slack_message(message) - logger.info(message) - except CustomError: - ref = fb_db.reference(f"v2/projectDrafts/{project_draft_id}") - ref.set({}) - newline = "\n" - message = ( - f"### PROJECT CREATION FAILED ###{newline}" - f'Project Name: {project_draft["name"]}{newline}' - f"Project Id: {project_draft_id}{newline}" - f"{newline}" - f"Project draft is deleted.{newline}" - f"Please check what went wrong." - ) - slack.send_slack_message(message) - slack.send_error(CustomError) - logger.exception(f"{project_draft_id} " f"- project creation failed") - continue - del fb_db - return created_project_ids - - -def _run_firebase_to_postgres(): - update_data.update_user_data() - update_data.update_project_data() - project_id_list = transfer_results.transfer_results() - - return project_id_list - - -def _run_generate_stats(project_id_list): - generate_stats.generate_stats(project_id_list) - - -def _run_generate_stats_all_projects(): - generate_stats.generate_stats_all_projects() - - -def _run_user_management(email, manager): - if manager is not None: - if manager: - user_management.set_project_manager_rights(email) - else: - user_management.remove_project_manager_rights(email) - - -cli.add_command(run_create_projects) -cli.add_command(run_firebase_to_postgres) -cli.add_command(run_generate_stats) -cli.add_command(run_generate_stats_all_projects) -cli.add_command(run_user_management) -cli.add_command(run_create_tutorial) -cli.add_command(run) + _run() From 3b8d8f37ced7234e6e90b793f971871ec517b6e2 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 14 Jan 2020 15:15:25 +0100 Subject: [PATCH 30/51] add parameter type FileType and add docstrings. --- .../mapswipe_workers/mapswipe_workers.py | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index 8f9652585..2bd3409e9 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -134,22 +134,20 @@ def run_generate_stats_all_projects() -> None: ) @click.option( "--manager", - help=( - f"Set option to grant or remove project manager credentials. " - f"Use true to grant credentials. " - f"Use false to remove credentials. " - ), + help=("Use true to grant credentials. Use false to remove credentials."), type=bool, ) def run_user_management(email, manager) -> None: + """ + Manage project manager credentials + + Grant or remove credentials. + """ try: - if email and manager: - if manager: - user_management.set_project_manager_rights(email) - else: - user_management.remove_project_manager_rights(email) - else: - click.echo(f"Please provide all required input arguments.") + if manager: + user_management.set_project_manager_rights(email) + elif not manager: + user_management.remove_project_manager_rights(email) except Exception as e: slack.send_error(e) sentry.capture_exception_sentry(e) @@ -158,12 +156,13 @@ def run_user_management(email, manager) -> None: @cli.command("create-tutorial") @click.option( - "--input_file", - help=(f"The json file with your tutorial information."), + "--input-file", + help=(f"A JSON file of the tutorial."), required=True, - type=str, + type=click.Path, ) def run_create_tutorial(input_file) -> None: + """Create a tutorial project from provided JSON file.""" try: logger.info(f"will generate tutorial based on {input_file}") with open(input_file) as json_file: From 30cd2d6faf465022278ff285852daef1288d9513 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Wed, 15 Jan 2020 10:29:17 +0100 Subject: [PATCH 31/51] Initalize slack, define slack messages and implement a send_slack_message helper function. --- .../mapswipe_workers/utils/slack_helper.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 mapswipe_workers/mapswipe_workers/utils/slack_helper.py diff --git a/mapswipe_workers/mapswipe_workers/utils/slack_helper.py b/mapswipe_workers/mapswipe_workers/utils/slack_helper.py new file mode 100644 index 000000000..c89d17772 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/utils/slack_helper.py @@ -0,0 +1,34 @@ +"""Initialize slack client with values provided by the config file""" + +import json +import slack + +from mapswipe_workers.definitions import CONFIG_PATH + + +def send_slack_message(message_type: str, project_name: str, project_id: str = None): + + with open(CONFIG_PATH) as config_file: + config = json.load(config_file) + slack_channel = config["slack"]["channel"] + slack_client = slack.WebClient(token=config["slack"]["token"]) + + if message_type == "success": + message = ( + "### PROJECT CREATION SUCCESSFUL ###\n" + + "Project Name: {0}\n".format(project_name) + + "Project Id: {0}\n\n".format(project_id) + + "Make sure to activate the project using the manager dashboard.\n" + + "Happy Swiping. :)" + ) + elif message_type == "fail": + message = ( + "### PROJECT CREATION FAILED ###\n" + + "Project Name: {0}\n".format(project_name) + + "Project draft is deleted." + ) + else: + # TODO: Raise an Exception + pass + + slack_client.chat_postMessage(channel=slack_channel, text=message) From 26f66ad1f9f5bc2906af32a22fdbdaeaf0cb0688 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Wed, 15 Jan 2020 10:30:39 +0100 Subject: [PATCH 32/51] init sentry and assign sentry_sdk to variable name sentry similar to logger. --- mapswipe_workers/mapswipe_workers/definitions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index b9b5be5f8..79aadbc35 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -1,7 +1,10 @@ +import json import logging import logging.config import os +import sentry_sdk + from mapswipe_workers.project_types.build_area.build_area_project import ( BuildAreaProject, ) @@ -37,6 +40,12 @@ logging.config.fileConfig(fname=LOGGING_CONFIG_PATH, disable_existing_loggers=True) logger = logging.getLogger("Mapswipe Workers") +with open(CONFIG_PATH) as config_file: + config = json.load(config_file) + sentry_url = config["sentry"]["dsn"] + sentry_sdk.init(sentry_url) +sentry = sentry_sdk + class CustomError(Exception): pass From 93aa16bd3eb30493d7b26c2b6121e38373fd3b67 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Wed, 15 Jan 2020 10:35:06 +0100 Subject: [PATCH 33/51] utilize slack helper module and definitions of sentry in definitions.py --- .../mapswipe_workers/mapswipe_workers.py | 67 ++++++------------- 1 file changed, 20 insertions(+), 47 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index 701549708..6b59b9042 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -4,14 +4,15 @@ import json import time -import click import schedule as sched + +import click from mapswipe_workers import auth from mapswipe_workers.definitions import ( PROJECT_TYPE_CLASSES, - PROJECT_TYPE_NAMES, CustomError, logger, + sentry, ) from mapswipe_workers.firebase_to_postgres import ( archive_project, @@ -20,10 +21,9 @@ ) from mapswipe_workers.generate_stats import generate_stats from mapswipe_workers.project_types.build_area import build_area_tutorial -from mapswipe_workers.project_types.change_detection import ( - change_detection_tutorial, -) -from mapswipe_workers.utils import sentry, slack, user_management +from mapswipe_workers.project_types.change_detection import change_detection_tutorial +from mapswipe_workers.utils import user_management +from mapswipe_workers.utils.slack_helper import send_slack_message class PythonLiteralOption(click.Option): @@ -63,6 +63,7 @@ def run_create_projects(): for project_draft_id, project_draft in project_drafts.items(): project_draft["projectDraftId"] = project_draft_id project_type = project_draft["projectType"] + project_name = project_draft["name"] try: # Create a project object using appropriate class (project type). project = PROJECT_TYPE_CLASSES[project_type](project_draft) @@ -71,37 +72,15 @@ def run_create_projects(): project.calc_required_results() # Save project and its groups and tasks to Firebase and Postgres. project.save_project() - newline = "\n" - message = ( - f"### PROJECT CREATION SUCCESSFUL ###{newline}" - f"Project Name: {project.name}{newline}" - f"Project Id: {project.projectId}{newline}" - f"Project Type: {PROJECT_TYPE_NAMES[project_type]}" - f"{newline}" - f"Make sure to activate the project " - f"using the manager dashboard." - f"{newline}" - f"Happy Swiping. :)" - ) - slack.send_slack_message(message) - logger.info(message) except CustomError: ref = fb_db.reference(f"v2/projectDrafts/{project_draft_id}") ref.set({}) - newline = "\n" - message = ( - f"### PROJECT CREATION FAILED ###{newline}" - f'Project Name: {project_draft["name"]}{newline}' - f"Project Id: {project_draft_id}{newline}" - f"{newline}" - f"Project draft is deleted.{newline}" - f"Please check what went wrong." - ) - slack.send_slack_message(message) - slack.send_error(CustomError) - logger.exception(f"{project_draft_id} " f"- project creation failed") + send_slack_message("fail", project_name, project.projectId) + logger.exception("Failed: Project Creation ({0}))".format(project_name)) sentry.capture_exception() - continue + send_slack_message("success", project_name, project.projectId) + logger.info("Success: Project Creation ({0})".format(project_name)) + continue @cli.command("firebase-to-postgres") @@ -154,10 +133,9 @@ def run_user_management(email, manager) -> None: user_management.set_project_manager_rights(email) elif not manager: user_management.remove_project_manager_rights(email) - except Exception as e: - slack.send_error(e) - sentry.capture_exception_sentry(e) - logger.exception(e) + except Exception: + logger.exception() + sentry.capture_exception() @cli.command("create-tutorial") @@ -181,20 +159,15 @@ def run_create_tutorial(input_file) -> None: 1: build_area_tutorial.create_tutorial, 3: change_detection_tutorial.create_tutorial, } - project_types_tutorial[project_type](tutorial) - except Exception as e: - slack.send_error(e) - sentry.capture_exception_sentry(e) - logger.exception(e) + except Exception: + logger.exception() + sentry.capture_exception() @click.command("archive") @click.option( - "--project-id", - "-i", - help=("Archive project with giving project id"), - type=str, + "--project-id", "-i", help=("Archive project with giving project id"), type=str, ) @click.option( "--project-ids", @@ -237,4 +210,4 @@ def _run(): sched.run_pending() time.sleep(1) else: - _run() \ No newline at end of file + _run() From 75deaafff1fcfc472d78d6a8bb0b8dec4beedb22 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Wed, 15 Jan 2020 10:35:33 +0100 Subject: [PATCH 34/51] renamed modules --- .../mapswipe_workers/utils/init_sentry.py | 12 ------------ .../mapswipe_workers/utils/init_slack.py | 12 ------------ 2 files changed, 24 deletions(-) delete mode 100644 mapswipe_workers/mapswipe_workers/utils/init_sentry.py delete mode 100644 mapswipe_workers/mapswipe_workers/utils/init_slack.py diff --git a/mapswipe_workers/mapswipe_workers/utils/init_sentry.py b/mapswipe_workers/mapswipe_workers/utils/init_sentry.py deleted file mode 100644 index 031904144..000000000 --- a/mapswipe_workers/mapswipe_workers/utils/init_sentry.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Initialize sentry client with values provided by the config file""" - -import json -import sentry_sdk - -from mapswipe_workers.definitions import CONFIG_PATH - - -with open(CONFIG_PATH) as config_file: - config = json.load(config_file) - sentry_url = config["sentry"]["dsn"] - sentry_sdk.init(sentry_url) diff --git a/mapswipe_workers/mapswipe_workers/utils/init_slack.py b/mapswipe_workers/mapswipe_workers/utils/init_slack.py deleted file mode 100644 index f4b504f90..000000000 --- a/mapswipe_workers/mapswipe_workers/utils/init_slack.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Initialize slack client with values provided by the config file""" - -import json -import slack - -from mapswipe_workers.definitions import CONFIG_PATH - - -with open(CONFIG_PATH) as config_file: - config = json.load(config_file) - slack_channel = config["slack"]["channel"] - slack_client = slack.WebClient(token=config["slack"]["token"]) From 106feab71970f61a6278920c84a25b6e1565b2a1 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Wed, 15 Jan 2020 11:33:25 +0100 Subject: [PATCH 35/51] Travis template. Run black and flake8. --- .travis | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .travis diff --git a/.travis b/.travis new file mode 100644 index 000000000..26bfee09e --- /dev/null +++ b/.travis @@ -0,0 +1,21 @@ +# This is a special configuration file to run tests on Travis-CI via +# GitHub notifications when changes are committed. +# +# For technical details, see http://travis-ci.org/ + +language: python +python: + - "3.6" + - "3.7" + - "3.8" + +before_install: + - pip install --upgrade pip setuptools + - pip install flake8 black + - echo "Using flake8 to check Python code." + - flake8 mapswipe_workers/mapswipe_workers/*.py + - echo "Using black to check Python code." + - black --check mapswipe_workers/mapswipe_workers/*.py + +install: + - pip install mapswipe_workers/ From 1e27457f8a8bed544cd3adff37b1308f0e18008d Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Wed, 15 Jan 2020 11:34:35 +0100 Subject: [PATCH 36/51] Travis template. Run black and flake8. --- .travis | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis b/.travis index 26bfee09e..89c409bd7 100644 --- a/.travis +++ b/.travis @@ -12,9 +12,7 @@ python: before_install: - pip install --upgrade pip setuptools - pip install flake8 black - - echo "Using flake8 to check Python code." - flake8 mapswipe_workers/mapswipe_workers/*.py - - echo "Using black to check Python code." - black --check mapswipe_workers/mapswipe_workers/*.py install: From e47742adbeddeeb3ba4b5fc0c0d54f075f49e638 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Thu, 16 Jan 2020 15:54:17 +0100 Subject: [PATCH 37/51] add configuration for flake8 --- .travis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis b/.travis index 89c409bd7..598a4c57b 100644 --- a/.travis +++ b/.travis @@ -12,7 +12,7 @@ python: before_install: - pip install --upgrade pip setuptools - pip install flake8 black - - flake8 mapswipe_workers/mapswipe_workers/*.py + - flake8 --config=mapswipe_workers/setup.cfg mapswipe_workers/mapswipe_workers/*.py - black --check mapswipe_workers/mapswipe_workers/*.py install: From 39f6f50a7d64105d5d175952233ddfb4a7835715 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 28 Jan 2020 09:52:33 +0100 Subject: [PATCH 38/51] Black --- mapswipe_workers/mapswipe_workers/auth.py | 78 +++++++++-------------- 1 file changed, 30 insertions(+), 48 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/auth.py b/mapswipe_workers/mapswipe_workers/auth.py index 560545ec1..264cb8ee3 100644 --- a/mapswipe_workers/mapswipe_workers/auth.py +++ b/mapswipe_workers/mapswipe_workers/auth.py @@ -30,30 +30,31 @@ def load_config(): def get_api_key(tileserver): CONFIG = load_config() try: - if tileserver == 'custom': + if tileserver == "custom": return None else: - return CONFIG['imagery'][tileserver]['api_key'] + return CONFIG["imagery"][tileserver]["api_key"] except KeyError: print( - f'Could not find the API key for imagery tileserver ' - f'{tileserver} in {CONFIG_PATH}.' - ) + f"Could not find the API key for imagery tileserver " + f"{tileserver} in {CONFIG_PATH}." + ) raise def get_tileserver_url(tileserver): CONFIG = load_config() try: - if tileserver == 'custom': + if tileserver == "custom": return None else: - return CONFIG['imagery'][tileserver]['url'] + return CONFIG["imagery"][tileserver]["url"] except KeyError: - print('Could not find the url for imagery tileserver {} in {}.'.format( - tileserver, - CONFIG_PATH - )) + print( + "Could not find the url for imagery tileserver {} in {}.".format( + tileserver, CONFIG_PATH + ) + ) raise @@ -76,13 +77,11 @@ def firebaseDB(): except ValueError: cred = credentials.Certificate(SERVICE_ACCOUNT_KEY_PATH) config = load_config() - databaseName = config['firebase']['database_name'] - databaseURL = f'https://{databaseName}.firebaseio.com' + databaseName = config["firebase"]["database_name"] + databaseURL = f"https://{databaseName}.firebaseio.com" # Initialize the app with a service account, granting admin privileges - firebase_admin.initialize_app(cred, { - 'databaseURL': databaseURL - }) + firebase_admin.initialize_app(cred, {"databaseURL": databaseURL}) # Return the imported Firebase Realtime Database module return db @@ -95,24 +94,19 @@ class postgresDB(object): def __init__(self): CONFIG = load_config() try: - host = CONFIG['postgres']['host'] - port = CONFIG['postgres']['port'] - dbname = CONFIG['postgres']['database'] - user = CONFIG['postgres']['username'] - password = CONFIG['postgres']['password'] + host = CONFIG["postgres"]["host"] + port = CONFIG["postgres"]["port"] + dbname = CONFIG["postgres"]["database"] + user = CONFIG["postgres"]["username"] + password = CONFIG["postgres"]["password"] except KeyError: raise Exception( - f'Could not load postgres credentials ' - f'from the configuration file' - ) + f"Could not load postgres credentials " f"from the configuration file" + ) self._db_connection = psycopg2.connect( - database=dbname, - user=user, - password=password, - host=host, - port=port - ) + database=dbname, user=user, password=password, host=host, port=port + ) def query(self, query, data=None): self._db_cur = self._db_connection.cursor() @@ -120,31 +114,19 @@ def query(self, query, data=None): self._db_connection.commit() self._db_cur.close() - def copy_from( - self, - f, - table, - columns - ): + def copy_from(self, f, table, columns): self._db_cur = self._db_connection.cursor() - self._db_cur.copy_from( - f, - table, - columns=columns - ) + self._db_cur.copy_from(f, table, columns=columns) self._db_connection.commit() self._db_cur.close() def copy_expert( - self, - sql, - file, - ): + self, sql, file, + ): self._db_cur = self._db_connection.cursor() self._db_cur.copy_expert( - sql, - file, - ) + sql, file, + ) self._db_connection.commit() self._db_cur.close() From 0d57903abde93f2cea290b901efc2267839846ab Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 28 Jan 2020 09:54:35 +0100 Subject: [PATCH 39/51] Remove unused function. --- mapswipe_workers/mapswipe_workers/auth.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/auth.py b/mapswipe_workers/mapswipe_workers/auth.py index 264cb8ee3..dd34a4851 100644 --- a/mapswipe_workers/mapswipe_workers/auth.py +++ b/mapswipe_workers/mapswipe_workers/auth.py @@ -58,16 +58,6 @@ def get_tileserver_url(tileserver): raise -def init_firebase(): - try: - # Is an App instance already initialized? - firebase_admin.get_app() - except ValueError: - cred = credentials.Certificate(SERVICE_ACCOUNT_KEY_PATH) - # Initialize the app with a service account, granting admin privileges - firebase_admin.initialize_app(cred) - - def firebaseDB(): try: # Is an App instance already initialized? From 928964cde3e426ee4f26eebc2f4c310cbe34aca5 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 28 Jan 2020 10:08:44 +0100 Subject: [PATCH 40/51] load config now definied in definitions.py. Import CONFIG dict from definitions. --- mapswipe_workers/mapswipe_workers/auth.py | 27 ++----------------- .../mapswipe_workers/definitions.py | 18 ++++++++++--- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/auth.py b/mapswipe_workers/mapswipe_workers/auth.py index dd34a4851..3b55efe3a 100644 --- a/mapswipe_workers/mapswipe_workers/auth.py +++ b/mapswipe_workers/mapswipe_workers/auth.py @@ -1,34 +1,14 @@ -#!/usr/bin/python3 -# -# Author: B. Herfort, M. Reinmuth, 2017 -############################################ - -import json - import psycopg2 import firebase_admin from firebase_admin import credentials from firebase_admin import db from mapswipe_workers.definitions import CONFIG_PATH +from mapswipe_workers.definitions import CONFIG from mapswipe_workers.definitions import SERVICE_ACCOUNT_KEY_PATH -def load_config(): - """ - Loads the user configuration values. - - Returns - ------- - dictonary - """ - with open(CONFIG_PATH) as f: - CONFIG = json.load(f) - return CONFIG - - def get_api_key(tileserver): - CONFIG = load_config() try: if tileserver == "custom": return None @@ -43,7 +23,6 @@ def get_api_key(tileserver): def get_tileserver_url(tileserver): - CONFIG = load_config() try: if tileserver == "custom": return None @@ -66,8 +45,7 @@ def firebaseDB(): return db except ValueError: cred = credentials.Certificate(SERVICE_ACCOUNT_KEY_PATH) - config = load_config() - databaseName = config["firebase"]["database_name"] + databaseName = CONFIG["firebase"]["database_name"] databaseURL = f"https://{databaseName}.firebaseio.com" # Initialize the app with a service account, granting admin privileges @@ -82,7 +60,6 @@ class postgresDB(object): _db_cur = None def __init__(self): - CONFIG = load_config() try: host = CONFIG["postgres"]["host"] port = CONFIG["postgres"]["port"] diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index b9b5be5f8..32135931b 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -1,3 +1,4 @@ +import json import logging import logging.config import os @@ -10,12 +11,25 @@ ) from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject + +class CustomError(Exception): + pass + + +def load_config(CONFIG_PATH) -> dict: + """Read the configuration file.""" + with open(CONFIG_PATH) as f: + return json.load(f) + + ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) CONFIG_DIR = os.path.abspath("/usr/share/config/mapswipe_workers/") CONFIG_PATH = os.path.join(CONFIG_DIR, "configuration.json") +CONFIG = load_config(CONFIG_PATH) + SERVICE_ACCOUNT_KEY_PATH = os.path.join(CONFIG_DIR, "serviceAccountKey.json") LOGGING_CONFIG_PATH = os.path.join(CONFIG_DIR, "logging.cfg") @@ -36,7 +50,3 @@ logging.config.fileConfig(fname=LOGGING_CONFIG_PATH, disable_existing_loggers=True) logger = logging.getLogger("Mapswipe Workers") - - -class CustomError(Exception): - pass From 33da93cae0a25ac7c3a39084d7fafd49e5ae6d4f Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 28 Jan 2020 10:15:42 +0100 Subject: [PATCH 41/51] Remove try-except statements where 'raise' is used in except-block. Add comments. Conform to style guidelines --- mapswipe_workers/mapswipe_workers/auth.py | 73 ++++++++--------------- 1 file changed, 24 insertions(+), 49 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/auth.py b/mapswipe_workers/mapswipe_workers/auth.py index 3b55efe3a..9c0fe0510 100644 --- a/mapswipe_workers/mapswipe_workers/auth.py +++ b/mapswipe_workers/mapswipe_workers/auth.py @@ -1,43 +1,25 @@ -import psycopg2 import firebase_admin -from firebase_admin import credentials -from firebase_admin import db +import psycopg2 +from firebase_admin import credentials, db -from mapswipe_workers.definitions import CONFIG_PATH -from mapswipe_workers.definitions import CONFIG -from mapswipe_workers.definitions import SERVICE_ACCOUNT_KEY_PATH +from mapswipe_workers.definitions import CONFIG, SERVICE_ACCOUNT_KEY_PATH -def get_api_key(tileserver): - try: - if tileserver == "custom": - return None - else: - return CONFIG["imagery"][tileserver]["api_key"] - except KeyError: - print( - f"Could not find the API key for imagery tileserver " - f"{tileserver} in {CONFIG_PATH}." - ) - raise +def get_api_key(tileserver: str) -> str: + if tileserver == "custom": + return None + else: + return CONFIG["imagery"][tileserver]["api_key"] -def get_tileserver_url(tileserver): - try: - if tileserver == "custom": - return None - else: - return CONFIG["imagery"][tileserver]["url"] - except KeyError: - print( - "Could not find the url for imagery tileserver {} in {}.".format( - tileserver, CONFIG_PATH - ) - ) - raise +def get_tileserver_url(tileserver: str) -> str: + if tileserver == "custom": + return None + else: + return CONFIG["imagery"][tileserver]["url"] -def firebaseDB(): +def firebaseDB() -> object: try: # Is an App instance already initialized? firebase_admin.get_app() @@ -56,23 +38,20 @@ def firebaseDB(): class postgresDB(object): + """Helper calss for Postgres interactions""" + _db_connection = None _db_cur = None def __init__(self): - try: - host = CONFIG["postgres"]["host"] - port = CONFIG["postgres"]["port"] - dbname = CONFIG["postgres"]["database"] - user = CONFIG["postgres"]["username"] - password = CONFIG["postgres"]["password"] - except KeyError: - raise Exception( - f"Could not load postgres credentials " f"from the configuration file" - ) + host = CONFIG["postgres"]["host"] + port = CONFIG["postgres"]["port"] + dbname = CONFIG["postgres"]["database"] + user = CONFIG["postgres"]["username"] + password = CONFIG["postgres"]["password"] self._db_connection = psycopg2.connect( - database=dbname, user=user, password=password, host=host, port=port + database=dbname, user=user, password=password, host=host, port=port, ) def query(self, query, data=None): @@ -87,13 +66,9 @@ def copy_from(self, f, table, columns): self._db_connection.commit() self._db_cur.close() - def copy_expert( - self, sql, file, - ): + def copy_expert(self, sql, file): self._db_cur = self._db_connection.cursor() - self._db_cur.copy_expert( - sql, file, - ) + self._db_cur.copy_expert(sql, file) self._db_connection.commit() self._db_cur.close() From 963fb0ca5f345ca1fb2551238c8bc039335fe67a Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 28 Jan 2020 15:03:53 +0100 Subject: [PATCH 42/51] 'import ogr/osr' is deprecated. 'from osgeo import ogr'is now used. --- .../generate_stats/tasking_manager_geometries.py | 2 +- .../project_types/build_area/build_area_project.py | 4 ++-- .../change_detection/change_detection_project.py | 4 ++-- .../project_types/footprint/footprint_project.py | 2 +- .../project_types/footprint/footprint_task.py | 2 +- .../project_types/footprint/grouping_functions.py | 2 +- .../mapswipe_workers/utils/geojson_functions.py | 6 ++++-- mapswipe_workers/mapswipe_workers/utils/tile_functions.py | 2 +- .../mapswipe_workers/utils/tile_grouping_functions.py | 2 +- 9 files changed, 14 insertions(+), 12 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/generate_stats/tasking_manager_geometries.py b/mapswipe_workers/mapswipe_workers/generate_stats/tasking_manager_geometries.py index a802eaf75..cbd1fd2d8 100644 --- a/mapswipe_workers/mapswipe_workers/generate_stats/tasking_manager_geometries.py +++ b/mapswipe_workers/mapswipe_workers/generate_stats/tasking_manager_geometries.py @@ -1,7 +1,7 @@ import csv from queue import Queue import threading -import ogr +from osgeo import ogr from mapswipe_workers.definitions import logger from mapswipe_workers.definitions import DATA_PATH diff --git a/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py b/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py index f79a546e6..7f5f02f31 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_project.py @@ -1,6 +1,6 @@ import os -import ogr -import osr +from osgeo import ogr +from osgeo import osr import json from mapswipe_workers.definitions import DATA_PATH diff --git a/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py b/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py index a97aec70f..f1f2ff0e0 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_project.py @@ -1,6 +1,6 @@ import os -import ogr -import osr +from osgeo import ogr +from osgeo import osr import json from mapswipe_workers.definitions import DATA_PATH diff --git a/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py b/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py index 579079036..b6c5f8dd3 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_project.py @@ -1,6 +1,6 @@ import os import urllib.request -import ogr +from osgeo import ogr from mapswipe_workers.definitions import DATA_PATH from mapswipe_workers.definitions import CustomError diff --git a/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_task.py b/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_task.py index 55323e2f8..7320a2c41 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_task.py +++ b/mapswipe_workers/mapswipe_workers/project_types/footprint/footprint_task.py @@ -1,4 +1,4 @@ -import ogr +from osgeo import ogr from mapswipe_workers.base.base_task import BaseTask diff --git a/mapswipe_workers/mapswipe_workers/project_types/footprint/grouping_functions.py b/mapswipe_workers/mapswipe_workers/project_types/footprint/grouping_functions.py index 1345bbb75..ec5f597b6 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/footprint/grouping_functions.py +++ b/mapswipe_workers/mapswipe_workers/project_types/footprint/grouping_functions.py @@ -1,4 +1,4 @@ -import ogr +from osgeo import ogr import argparse import json diff --git a/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py b/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py index 4d1913311..34aeac155 100644 --- a/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py +++ b/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py @@ -1,8 +1,10 @@ import json import os import subprocess -import osr -import ogr + +from osgeo import ogr +from osgeo import osr + from mapswipe_workers.definitions import logger diff --git a/mapswipe_workers/mapswipe_workers/utils/tile_functions.py b/mapswipe_workers/mapswipe_workers/utils/tile_functions.py index e059710c6..0c26d67db 100644 --- a/mapswipe_workers/mapswipe_workers/utils/tile_functions.py +++ b/mapswipe_workers/mapswipe_workers/utils/tile_functions.py @@ -1,5 +1,5 @@ import math -import ogr +from osgeo import ogr class Point: diff --git a/mapswipe_workers/mapswipe_workers/utils/tile_grouping_functions.py b/mapswipe_workers/mapswipe_workers/utils/tile_grouping_functions.py index c41af1390..18075abeb 100644 --- a/mapswipe_workers/mapswipe_workers/utils/tile_grouping_functions.py +++ b/mapswipe_workers/mapswipe_workers/utils/tile_grouping_functions.py @@ -1,5 +1,5 @@ import math -import ogr +from osgeo import ogr from mapswipe_workers.definitions import logger from mapswipe_workers.utils import tile_functions as t From fb0c999961ffe39b556edd4f73dd114b44585382 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Tue, 28 Jan 2020 17:04:52 +0100 Subject: [PATCH 43/51] Use official gdal docker image from osgeo. --- mapswipe_workers/Dockerfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mapswipe_workers/Dockerfile b/mapswipe_workers/Dockerfile index 5d2d83c2b..eb0a204e3 100644 --- a/mapswipe_workers/Dockerfile +++ b/mapswipe_workers/Dockerfile @@ -1,12 +1,12 @@ -# This image contains Python 3 and the latest pre-compiled version of GDAL -# Based on Debian (Including apt-get update && upgrade) -FROM thinkwhere/gdal-python:3.7-shippable +# https://github.com/OSGeo/gdal/tree/master/gdal/docker +# Image includes python3.6, gdal-python, gdal-bin +FROM osgeo/gdal:ubuntu-small-latest -# Install gdal-bin to get ogr2ogr tool +# Install pip RUN apt-get update -RUN apt-get --yes install gdal-bin +RUN apt-get --yes install python3-pip -# create directories for config, logs and data +# Create directories for config, logs and data ARG config_dir=/usr/share/config/mapswipe_workers/ ARG repo_dir=/usr/local/mapswipe_workers/ ARG data_dir=/var/lib/mapswipe_workers/ @@ -26,7 +26,7 @@ RUN mkdir -p $data_dir"api-data/tasks/" RUN mkdir -p $data_dir"api-data/yes_maybe/" RUN mkdir -p $data_dir"api-data/hot_tm/" -# copy mapswipe workers repo from local repo +# Copy mapswipe workers repo from local repo WORKDIR $repo_dir COPY mapswipe_workers/ mapswipe_workers/ COPY sample_data/ sample_data/ @@ -35,8 +35,8 @@ COPY requirements.txt . COPY setup.py . COPY config $config_dir -# Install dependencies and mapswipe-workers -# RUN python setup.py install -RUN pip install . +# Update setuptools and install mapswipe-workers with dependencies (requirements.txt) +RUN pip3 install --upgrade setuptools +RUN pip3 install . -# we don't use a CMD here, this will be defined in docker-compose.yaml +# Don't use a CMD here, this will be defined in docker-compose.yaml From 92a5751dcdd928b46c4ea734bb1aa59e6a90bd91 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Thu, 30 Jan 2020 09:16:15 +0100 Subject: [PATCH 44/51] avoid circular imports --- .../mapswipe_workers/definitions.py | 20 -------------- .../mapswipe_workers/mapswipe_workers.py | 26 ++++++++++++------- 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index 1ef7d35d3..b0d3e3010 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -5,14 +5,6 @@ import sentry_sdk -from mapswipe_workers.project_types.build_area.build_area_project import ( - BuildAreaProject, -) -from mapswipe_workers.project_types.change_detection.change_detection_project import ( - ChangeDetectionProject, -) -from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject - class CustomError(Exception): pass @@ -38,18 +30,6 @@ def load_config(CONFIG_PATH) -> dict: DATA_PATH = os.path.abspath("/var/lib/mapswipe_workers/") -PROJECT_TYPE_CLASSES = { - 1: BuildAreaProject, - 2: FootprintProject, - 3: ChangeDetectionProject, -} - -PROJECT_TYPE_NAMES = { - 1: BuildAreaProject.project_type_name, - 2: FootprintProject.project_type_name, - 3: ChangeDetectionProject.project_type_name, -} - logging.config.fileConfig(fname=LOGGING_CONFIG_PATH, disable_existing_loggers=True) logger = logging.getLogger("Mapswipe Workers") diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index 6b59b9042..bc4c725cc 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -4,16 +4,10 @@ import json import time -import schedule as sched - import click +import schedule as sched from mapswipe_workers import auth -from mapswipe_workers.definitions import ( - PROJECT_TYPE_CLASSES, - CustomError, - logger, - sentry, -) +from mapswipe_workers.definitions import CustomError, logger, sentry from mapswipe_workers.firebase_to_postgres import ( archive_project, transfer_results, @@ -21,7 +15,14 @@ ) from mapswipe_workers.generate_stats import generate_stats from mapswipe_workers.project_types.build_area import build_area_tutorial +from mapswipe_workers.project_types.build_area.build_area_project import ( + BuildAreaProject, +) from mapswipe_workers.project_types.change_detection import change_detection_tutorial +from mapswipe_workers.project_types.change_detection.change_detection_project import ( + ChangeDetectionProject, +) +from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject from mapswipe_workers.utils import user_management from mapswipe_workers.utils.slack_helper import send_slack_message @@ -53,6 +54,13 @@ def run_create_projects(): Create projects with groups and tasks. Save created projects, groups and tasks to Firebase and Postgres. """ + + project_type_classes = { + 1: BuildAreaProject, + 2: FootprintProject, + 3: ChangeDetectionProject, + } + fb_db = auth.firebaseDB() ref = fb_db.reference("v2/projectDrafts/") project_drafts = ref.get() @@ -66,7 +74,7 @@ def run_create_projects(): project_name = project_draft["name"] try: # Create a project object using appropriate class (project type). - project = PROJECT_TYPE_CLASSES[project_type](project_draft) + project = project_type_classes[project_type](project_draft) project.geometry = project.validate_geometries() project.create_groups() project.calc_required_results() From 5b864edf0c7f0366b2cfa319cd29754c3d8e09fc Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Thu, 30 Jan 2020 09:23:42 +0100 Subject: [PATCH 45/51] Correct wrong import location. --- .../firebase_to_postgres/transfer_results.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py index 608a9c9d3..fd91f688c 100644 --- a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py +++ b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py @@ -4,9 +4,8 @@ import dateutil.parser from mapswipe_workers import auth -from mapswipe_workers.definitions import logger +from mapswipe_workers.definitions import logger, sentry from mapswipe_workers.firebase_to_postgres import update_data -from mapswipe_workers.utils import sentry def transfer_results(project_id_list=None): @@ -224,4 +223,4 @@ def get_projects_from_postgres(): project_ids = [i[0] for i in raw_ids] del pg_db - return project_ids \ No newline at end of file + return project_ids From 00b1e72cc87e16d0d71d1a7b72c115e2df46c73d Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Thu, 30 Jan 2020 09:28:51 +0100 Subject: [PATCH 46/51] Correct wrong import location. --- .../mapswipe_workers/utils/user_management.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/user_management.py b/mapswipe_workers/mapswipe_workers/utils/user_management.py index 8cf5d72ed..33e272613 100644 --- a/mapswipe_workers/mapswipe_workers/utils/user_management.py +++ b/mapswipe_workers/mapswipe_workers/utils/user_management.py @@ -5,9 +5,11 @@ from requests.exceptions import HTTPError from mapswipe_workers.auth import firebaseDB -from mapswipe_workers.definitions import CustomError -from mapswipe_workers.definitions import logger -from mapswipe_workers.auth import load_config +from mapswipe_workers.definitions import ( + CustomError, + logger, + CONFIG + ) def set_project_manager_rights(email): @@ -92,8 +94,7 @@ def delete_user(email): def sign_in_with_email_and_password(email, password): - config = load_config() - api_key = config['firebase']['api_key'] + api_key = CONFIG['firebase']['api_key'] request_ref = "https://www.googleapis.com/identitytoolkit/v3/relyingparty/verifyPassword?key={0}".format(api_key) headers = {"content-type": "application/json; charset=UTF-8"} data = json.dumps({"email": email, "password": password, "returnSecureToken": True}) @@ -104,8 +105,7 @@ def sign_in_with_email_and_password(email, password): def get_firebase_db(path, custom_arguments=None, token=None): - config = load_config() - databaseName = config['firebase']['database_name'] + databaseName = CONFIG['firebase']['database_name'] database_url = f'https://{databaseName}.firebaseio.com' request_ref = '{0}{1}.json?{3}auth={2}'.format(database_url, path, token, custom_arguments) headers = {"content-type": "application/json; charset=UTF-8"} @@ -122,8 +122,7 @@ def get_firebase_db(path, custom_arguments=None, token=None): def set_firebase_db(path, data, token=None): - config = load_config() - databaseName = config['firebase']['database_name'] + databaseName = CONFIG['firebase']['database_name'] database_url = f'https://{databaseName}.firebaseio.com' request_ref = '{0}{1}.json?auth={2}'.format(database_url, path, token) headers = {"content-type": "application/json; charset=UTF-8"} @@ -141,8 +140,7 @@ def set_firebase_db(path, data, token=None): def update_firebase_db(path, data, token=None): - config = load_config() - databaseName = config['firebase']['database_name'] + databaseName = CONFIG['firebase']['database_name'] database_url = f'https://{databaseName}.firebaseio.com' request_ref = '{0}{1}.json?auth={2}'.format(database_url, path, token) headers = {"content-type": "application/json; charset=UTF-8"} From 797f8a8022597bc789bdf22aef6360a34b375c23 Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Fri, 31 Jan 2020 13:24:02 +0100 Subject: [PATCH 47/51] Change arguement type to string due to unresolved error when using click.Path type: AssertionError: Attempted to use an uninstantiated parameter type --- mapswipe_workers/mapswipe_workers/mapswipe_workers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index bc4c725cc..176bb3612 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -151,7 +151,7 @@ def run_user_management(email, manager) -> None: "--input-file", help=(f"A JSON file of the tutorial."), required=True, - type=click.Path, + type=str, ) def run_create_tutorial(input_file) -> None: """Create a tutorial project from provided JSON file.""" From 4f347537c0934735eb16a8ce7aab13bafab4934b Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Fri, 31 Jan 2020 13:38:46 +0100 Subject: [PATCH 48/51] add archive and run commands to cli group of commands --- mapswipe_workers/mapswipe_workers/mapswipe_workers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index 176bb3612..0e501ea72 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -173,7 +173,7 @@ def run_create_tutorial(input_file) -> None: sentry.capture_exception() -@click.command("archive") +@cli.command("archive") @click.option( "--project-id", "-i", help=("Archive project with giving project id"), type=str, ) @@ -195,7 +195,7 @@ def run_archive_project(project_id, project_ids): archive_project.archive_project(project_ids) -@click.command("run") +@cli.command("run") @click.option( "--schedule", is_flag=True, help=("Schedule jobs to run every 10 minutes.") ) From 843c60d928b7f5c418ba72d4c9b0b8e2804078ec Mon Sep 17 00:00:00 2001 From: matthias_schaub Date: Fri, 31 Jan 2020 13:50:09 +0100 Subject: [PATCH 49/51] Set C.UTF-8 locale as default needed by the Click library. --- mapswipe_workers/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mapswipe_workers/Dockerfile b/mapswipe_workers/Dockerfile index eb0a204e3..bfc42a3ea 100644 --- a/mapswipe_workers/Dockerfile +++ b/mapswipe_workers/Dockerfile @@ -2,6 +2,9 @@ # Image includes python3.6, gdal-python, gdal-bin FROM osgeo/gdal:ubuntu-small-latest +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + # Install pip RUN apt-get update RUN apt-get --yes install python3-pip From 49f5d9985525cab9c1d5e09aea217bbe19608029 Mon Sep 17 00:00:00 2001 From: h9b Date: Sat, 1 Feb 2020 13:34:01 +0100 Subject: [PATCH 50/51] fix docker logging issue #277 --- .../mapswipe_workers/mapswipe_workers.py | 54 ++++++++++++++----- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index 0e501ea72..9d7d23a68 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -37,8 +37,8 @@ def type_cast_value(self, ctx, value): @click.group() -@click.option("--verbose", "-v", is_flag=True, help="Enable logging.") @click.version_option() +@click.option("--verbose", "-v", is_flag=True, help="Enable logging.") def cli(verbose): """Enable logging.""" if not verbose: @@ -47,6 +47,16 @@ def cli(verbose): @cli.command("create-projects") def run_create_projects(): + """ + This is the wrapper function to create projects from submitted projects drafts. + We do it this way, to be able to use --verbose flag + for the _run_create_projects function. + Otherwise we can't use --verbose during run function. + """ + _run_create_projects() + + +def _run_create_projects(): """ Create projects from submitted project drafts. @@ -66,6 +76,7 @@ def run_create_projects(): project_drafts = ref.get() if project_drafts is None: + logger.info("There are no project drafts in firebase.") return None for project_draft_id, project_draft in project_drafts.items(): @@ -93,6 +104,17 @@ def run_create_projects(): @cli.command("firebase-to-postgres") def run_firebase_to_postgres() -> list: + """ + This is the wrapper function to update users and + transfer results from Firebase to Postgres. + We do it this way, to be able to use --verbose flag + for the _run_firebase_to_postgres function. + Otherwise we can't use --verbose during run function. + """ + return _run_firebase_to_postgres() + + +def _run_firebase_to_postgres() -> list: """Update users and transfer results from Firebase to Postgres.""" update_data.update_user_data() update_data.update_project_data() @@ -107,10 +129,20 @@ def run_firebase_to_postgres() -> list: default="[]", help=( "Project ids for which to generate stats as a list of strings: " - + """["project_a", "project_b"]""" + """ '["project_a", "project_b"]' """ ), ) def run_generate_stats(project_ids: list) -> None: + """ + This is the wrapper function to generate statistics for given project ids. + We do it this way, to be able to use --verbose flag + for the _run_generate_stats function. + Otherwise we can't use --verbose during run function. + """ + _run_generate_stats(project_ids) + + +def _run_generate_stats(project_ids: list) -> None: """Generate statistics for given project ids.""" generate_stats.generate_stats(project_ids) @@ -148,10 +180,7 @@ def run_user_management(email, manager) -> None: @cli.command("create-tutorial") @click.option( - "--input-file", - help=(f"A JSON file of the tutorial."), - required=True, - type=str, + "--input-file", help=(f"A JSON file of the tutorial."), required=True, type=str, ) def run_create_tutorial(input_file) -> None: """Create a tutorial project from provided JSON file.""" @@ -196,10 +225,8 @@ def run_archive_project(project_id, project_ids): @cli.command("run") -@click.option( - "--schedule", is_flag=True, help=("Schedule jobs to run every 10 minutes.") -) -def run(schedule: bool) -> None: +@click.option("--schedule", is_flag=True, help="Schedule jobs to run every 10 minutes.") +def run(schedule): """ Run all commands. @@ -208,9 +235,10 @@ def run(schedule: bool) -> None: """ def _run(): - run_create_projects() - project_ids = run_firebase_to_postgres() - run_generate_stats(project_ids) + logger.info("start mapswipe backend workflow.") + _run_create_projects() + project_ids = _run_firebase_to_postgres() + _run_generate_stats(project_ids) if schedule: sched.every(10).minutes.do(_run) From 8b9a3ae13a456b8cc12523552f250a683e193dae Mon Sep 17 00:00:00 2001 From: h9b Date: Sat, 1 Feb 2020 18:42:44 +0100 Subject: [PATCH 51/51] call run method of scheduled task --- mapswipe_workers/mapswipe_workers/mapswipe_workers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index 9d7d23a68..2152cc649 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -241,7 +241,7 @@ def _run(): _run_generate_stats(project_ids) if schedule: - sched.every(10).minutes.do(_run) + sched.every(10).minutes.do(_run).run() while True: sched.run_pending() time.sleep(1)