Circular imports

mapswipe_workers/Dockerfile

@@ -1,12 +1,15 @@
-# This image contains Python 3 and the latest pre-compiled version of GDAL
-# Based on Debian (Including apt-get update && upgrade)
-FROM thinkwhere/gdal-python:3.7-shippable
+# https://github.com/OSGeo/gdal/tree/master/gdal/docker
+# Image includes python3.6, gdal-python, gdal-bin
+FROM osgeo/gdal:ubuntu-small-latest
 
-# Install gdal-bin to get ogr2ogr tool
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+# Install pip
 RUN apt-get update
-RUN apt-get --yes install gdal-bin
+RUN apt-get --yes install python3-pip
 
-# create directories for config, logs and data
+# Create directories for config, logs and data
 ARG config_dir=/usr/share/config/mapswipe_workers/
 ARG repo_dir=/usr/local/mapswipe_workers/
 ARG data_dir=/var/lib/mapswipe_workers/
@@ -26,7 +29,7 @@ RUN mkdir -p $data_dir"api-data/tasks/"
 RUN mkdir -p $data_dir"api-data/yes_maybe/"
 RUN mkdir -p $data_dir"api-data/hot_tm/"
 
-# copy mapswipe workers repo from local repo
+# Copy mapswipe workers repo from local repo
 WORKDIR $repo_dir
 COPY mapswipe_workers/ mapswipe_workers/
 COPY sample_data/ sample_data/
@@ -35,8 +38,8 @@ COPY requirements.txt .
 COPY setup.py .
 COPY config $config_dir
 
-# Install dependencies and mapswipe-workers
-# RUN python setup.py install
-RUN pip install .
+# Update setuptools and install mapswipe-workers with dependencies (requirements.txt)
+RUN pip3 install --upgrade setuptools
+RUN pip3 install .
 
-# we don't use a CMD here, this will be defined in docker-compose.yaml
+# Don't use a CMD here, this will be defined in docker-compose.yaml

mapswipe_workers/mapswipe_workers/definitions.py

@@ -5,14 +5,6 @@
 
 import sentry_sdk
 
-from mapswipe_workers.project_types.build_area.build_area_project import (
-    BuildAreaProject,
-)
-from mapswipe_workers.project_types.change_detection.change_detection_project import (
-    ChangeDetectionProject,
-)
-from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject
-
 
 class CustomError(Exception):
     pass
@@ -38,18 +30,6 @@ def load_config(CONFIG_PATH) -> dict:
 
 DATA_PATH = os.path.abspath("/var/lib/mapswipe_workers/")
 
-PROJECT_TYPE_CLASSES = {
-    1: BuildAreaProject,
-    2: FootprintProject,
-    3: ChangeDetectionProject,
-}
-
-PROJECT_TYPE_NAMES = {
-    1: BuildAreaProject.project_type_name,
-    2: FootprintProject.project_type_name,
-    3: ChangeDetectionProject.project_type_name,
-}
-
 logging.config.fileConfig(fname=LOGGING_CONFIG_PATH, disable_existing_loggers=True)
 logger = logging.getLogger("Mapswipe Workers")
 

mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py

@@ -4,9 +4,8 @@
 import dateutil.parser
 
 from mapswipe_workers import auth
-from mapswipe_workers.definitions import logger
+from mapswipe_workers.definitions import logger, sentry
 from mapswipe_workers.firebase_to_postgres import update_data
-from mapswipe_workers.utils import sentry
 
 
 def transfer_results(project_id_list=None):
@@ -224,4 +223,4 @@ def get_projects_from_postgres():
     project_ids = [i[0] for i in raw_ids]
 
     del pg_db
-    return project_ids
+    return project_ids

mapswipe_workers/mapswipe_workers/mapswipe_workers.py

@@ -4,24 +4,25 @@
 import json
 import time
 
-import schedule as sched
-
 import click
+import schedule as sched
 from mapswipe_workers import auth
-from mapswipe_workers.definitions import (
-    PROJECT_TYPE_CLASSES,
-    CustomError,
-    logger,
-    sentry,
-)
+from mapswipe_workers.definitions import CustomError, logger, sentry
 from mapswipe_workers.firebase_to_postgres import (
     archive_project,
     transfer_results,
     update_data,
 )
 from mapswipe_workers.generate_stats import generate_stats
 from mapswipe_workers.project_types.build_area import build_area_tutorial
+from mapswipe_workers.project_types.build_area.build_area_project import (
+    BuildAreaProject,
+)
 from mapswipe_workers.project_types.change_detection import change_detection_tutorial
+from mapswipe_workers.project_types.change_detection.change_detection_project import (
+    ChangeDetectionProject,
+)
+from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject
 from mapswipe_workers.utils import user_management
 from mapswipe_workers.utils.slack_helper import send_slack_message
 
@@ -53,6 +54,13 @@ def run_create_projects():
     Create projects with groups and tasks.
     Save created projects, groups and tasks to Firebase and Postgres.
     """
+
+    project_type_classes = {
+        1: BuildAreaProject,
+        2: FootprintProject,
+        3: ChangeDetectionProject,
+    }
+
     fb_db = auth.firebaseDB()
     ref = fb_db.reference("v2/projectDrafts/")
     project_drafts = ref.get()
@@ -66,7 +74,7 @@ def run_create_projects():
         project_name = project_draft["name"]
         try:
             # Create a project object using appropriate class (project type).
-            project = PROJECT_TYPE_CLASSES[project_type](project_draft)
+            project = project_type_classes[project_type](project_draft)
             project.geometry = project.validate_geometries()
             project.create_groups()
             project.calc_required_results()
@@ -143,7 +151,7 @@ def run_user_management(email, manager) -> None:
     "--input-file",
     help=(f"A JSON file of the tutorial."),
     required=True,
-    type=click.Path,
+    type=str,
 )
 def run_create_tutorial(input_file) -> None:
     """Create a tutorial project from provided JSON file."""
@@ -165,7 +173,7 @@ def run_create_tutorial(input_file) -> None:
         sentry.capture_exception()
 
 
-@click.command("archive")
+@cli.command("archive")
 @click.option(
     "--project-id", "-i", help=("Archive project with giving project id"), type=str,
 )
@@ -187,7 +195,7 @@ def run_archive_project(project_id, project_ids):
     archive_project.archive_project(project_ids)
 
 
-@click.command("run")
+@cli.command("run")
 @click.option(
     "--schedule", is_flag=True, help=("Schedule jobs to run every 10 minutes.")
 )

mapswipe_workers/mapswipe_workers/utils/user_management.py

@@ -5,9 +5,11 @@
 from requests.exceptions import HTTPError
 
 from mapswipe_workers.auth import firebaseDB
-from mapswipe_workers.definitions import CustomError
-from mapswipe_workers.definitions import logger
-from mapswipe_workers.auth import load_config
+from mapswipe_workers.definitions import (
+        CustomError,
+        logger,
+        CONFIG
+        )
 
 
 def set_project_manager_rights(email):
@@ -92,8 +94,7 @@ def delete_user(email):
 
 
 def sign_in_with_email_and_password(email, password):
-    config = load_config()
-    api_key = config['firebase']['api_key']
+    api_key = CONFIG['firebase']['api_key']
     request_ref = "https://www.googleapis.com/identitytoolkit/v3/relyingparty/verifyPassword?key={0}".format(api_key)
     headers = {"content-type": "application/json; charset=UTF-8"}
     data = json.dumps({"email": email, "password": password, "returnSecureToken": True})
@@ -104,8 +105,7 @@ def sign_in_with_email_and_password(email, password):
 
 
 def get_firebase_db(path, custom_arguments=None, token=None):
-    config = load_config()
-    databaseName = config['firebase']['database_name']
+    databaseName = CONFIG['firebase']['database_name']
     database_url = f'https://{databaseName}.firebaseio.com'
     request_ref = '{0}{1}.json?{3}auth={2}'.format(database_url, path, token, custom_arguments)
     headers = {"content-type": "application/json; charset=UTF-8"}
@@ -122,8 +122,7 @@ def get_firebase_db(path, custom_arguments=None, token=None):
 
 
 def set_firebase_db(path, data, token=None):
-    config = load_config()
-    databaseName = config['firebase']['database_name']
+    databaseName = CONFIG['firebase']['database_name']
     database_url = f'https://{databaseName}.firebaseio.com'
     request_ref = '{0}{1}.json?auth={2}'.format(database_url, path, token)
     headers = {"content-type": "application/json; charset=UTF-8"}
@@ -141,8 +140,7 @@ def set_firebase_db(path, data, token=None):
 
 
 def update_firebase_db(path, data, token=None):
-    config = load_config()
-    databaseName = config['firebase']['database_name']
+    databaseName = CONFIG['firebase']['database_name']
     database_url = f'https://{databaseName}.firebaseio.com'
     request_ref = '{0}{1}.json?auth={2}'.format(database_url, path, token)
     headers = {"content-type": "application/json; charset=UTF-8"}