diff --git a/mapswipe_workers/Dockerfile b/mapswipe_workers/Dockerfile index 5d2d83c2b..bfc42a3ea 100644 --- a/mapswipe_workers/Dockerfile +++ b/mapswipe_workers/Dockerfile @@ -1,12 +1,15 @@ -# This image contains Python 3 and the latest pre-compiled version of GDAL -# Based on Debian (Including apt-get update && upgrade) -FROM thinkwhere/gdal-python:3.7-shippable +# https://github.com/OSGeo/gdal/tree/master/gdal/docker +# Image includes python3.6, gdal-python, gdal-bin +FROM osgeo/gdal:ubuntu-small-latest -# Install gdal-bin to get ogr2ogr tool +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + +# Install pip RUN apt-get update -RUN apt-get --yes install gdal-bin +RUN apt-get --yes install python3-pip -# create directories for config, logs and data +# Create directories for config, logs and data ARG config_dir=/usr/share/config/mapswipe_workers/ ARG repo_dir=/usr/local/mapswipe_workers/ ARG data_dir=/var/lib/mapswipe_workers/ @@ -26,7 +29,7 @@ RUN mkdir -p $data_dir"api-data/tasks/" RUN mkdir -p $data_dir"api-data/yes_maybe/" RUN mkdir -p $data_dir"api-data/hot_tm/" -# copy mapswipe workers repo from local repo +# Copy mapswipe workers repo from local repo WORKDIR $repo_dir COPY mapswipe_workers/ mapswipe_workers/ COPY sample_data/ sample_data/ @@ -35,8 +38,8 @@ COPY requirements.txt . COPY setup.py . COPY config $config_dir -# Install dependencies and mapswipe-workers -# RUN python setup.py install -RUN pip install . +# Update setuptools and install mapswipe-workers with dependencies (requirements.txt) +RUN pip3 install --upgrade setuptools +RUN pip3 install . -# we don't use a CMD here, this will be defined in docker-compose.yaml +# Don't use a CMD here, this will be defined in docker-compose.yaml diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index 1ef7d35d3..b0d3e3010 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -5,14 +5,6 @@ import sentry_sdk -from mapswipe_workers.project_types.build_area.build_area_project import ( - BuildAreaProject, -) -from mapswipe_workers.project_types.change_detection.change_detection_project import ( - ChangeDetectionProject, -) -from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject - class CustomError(Exception): pass @@ -38,18 +30,6 @@ def load_config(CONFIG_PATH) -> dict: DATA_PATH = os.path.abspath("/var/lib/mapswipe_workers/") -PROJECT_TYPE_CLASSES = { - 1: BuildAreaProject, - 2: FootprintProject, - 3: ChangeDetectionProject, -} - -PROJECT_TYPE_NAMES = { - 1: BuildAreaProject.project_type_name, - 2: FootprintProject.project_type_name, - 3: ChangeDetectionProject.project_type_name, -} - logging.config.fileConfig(fname=LOGGING_CONFIG_PATH, disable_existing_loggers=True) logger = logging.getLogger("Mapswipe Workers") diff --git a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py index 608a9c9d3..fd91f688c 100644 --- a/mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py +++ b/mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py @@ -4,9 +4,8 @@ import dateutil.parser from mapswipe_workers import auth -from mapswipe_workers.definitions import logger +from mapswipe_workers.definitions import logger, sentry from mapswipe_workers.firebase_to_postgres import update_data -from mapswipe_workers.utils import sentry def transfer_results(project_id_list=None): @@ -224,4 +223,4 @@ def get_projects_from_postgres(): project_ids = [i[0] for i in raw_ids] del pg_db - return project_ids \ No newline at end of file + return project_ids diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index 6b59b9042..0e501ea72 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -4,16 +4,10 @@ import json import time -import schedule as sched - import click +import schedule as sched from mapswipe_workers import auth -from mapswipe_workers.definitions import ( - PROJECT_TYPE_CLASSES, - CustomError, - logger, - sentry, -) +from mapswipe_workers.definitions import CustomError, logger, sentry from mapswipe_workers.firebase_to_postgres import ( archive_project, transfer_results, @@ -21,7 +15,14 @@ ) from mapswipe_workers.generate_stats import generate_stats from mapswipe_workers.project_types.build_area import build_area_tutorial +from mapswipe_workers.project_types.build_area.build_area_project import ( + BuildAreaProject, +) from mapswipe_workers.project_types.change_detection import change_detection_tutorial +from mapswipe_workers.project_types.change_detection.change_detection_project import ( + ChangeDetectionProject, +) +from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject from mapswipe_workers.utils import user_management from mapswipe_workers.utils.slack_helper import send_slack_message @@ -53,6 +54,13 @@ def run_create_projects(): Create projects with groups and tasks. Save created projects, groups and tasks to Firebase and Postgres. """ + + project_type_classes = { + 1: BuildAreaProject, + 2: FootprintProject, + 3: ChangeDetectionProject, + } + fb_db = auth.firebaseDB() ref = fb_db.reference("v2/projectDrafts/") project_drafts = ref.get() @@ -66,7 +74,7 @@ def run_create_projects(): project_name = project_draft["name"] try: # Create a project object using appropriate class (project type). - project = PROJECT_TYPE_CLASSES[project_type](project_draft) + project = project_type_classes[project_type](project_draft) project.geometry = project.validate_geometries() project.create_groups() project.calc_required_results() @@ -143,7 +151,7 @@ def run_user_management(email, manager) -> None: "--input-file", help=(f"A JSON file of the tutorial."), required=True, - type=click.Path, + type=str, ) def run_create_tutorial(input_file) -> None: """Create a tutorial project from provided JSON file.""" @@ -165,7 +173,7 @@ def run_create_tutorial(input_file) -> None: sentry.capture_exception() -@click.command("archive") +@cli.command("archive") @click.option( "--project-id", "-i", help=("Archive project with giving project id"), type=str, ) @@ -187,7 +195,7 @@ def run_archive_project(project_id, project_ids): archive_project.archive_project(project_ids) -@click.command("run") +@cli.command("run") @click.option( "--schedule", is_flag=True, help=("Schedule jobs to run every 10 minutes.") ) diff --git a/mapswipe_workers/mapswipe_workers/utils/user_management.py b/mapswipe_workers/mapswipe_workers/utils/user_management.py index 8cf5d72ed..33e272613 100644 --- a/mapswipe_workers/mapswipe_workers/utils/user_management.py +++ b/mapswipe_workers/mapswipe_workers/utils/user_management.py @@ -5,9 +5,11 @@ from requests.exceptions import HTTPError from mapswipe_workers.auth import firebaseDB -from mapswipe_workers.definitions import CustomError -from mapswipe_workers.definitions import logger -from mapswipe_workers.auth import load_config +from mapswipe_workers.definitions import ( + CustomError, + logger, + CONFIG + ) def set_project_manager_rights(email): @@ -92,8 +94,7 @@ def delete_user(email): def sign_in_with_email_and_password(email, password): - config = load_config() - api_key = config['firebase']['api_key'] + api_key = CONFIG['firebase']['api_key'] request_ref = "https://www.googleapis.com/identitytoolkit/v3/relyingparty/verifyPassword?key={0}".format(api_key) headers = {"content-type": "application/json; charset=UTF-8"} data = json.dumps({"email": email, "password": password, "returnSecureToken": True}) @@ -104,8 +105,7 @@ def sign_in_with_email_and_password(email, password): def get_firebase_db(path, custom_arguments=None, token=None): - config = load_config() - databaseName = config['firebase']['database_name'] + databaseName = CONFIG['firebase']['database_name'] database_url = f'https://{databaseName}.firebaseio.com' request_ref = '{0}{1}.json?{3}auth={2}'.format(database_url, path, token, custom_arguments) headers = {"content-type": "application/json; charset=UTF-8"} @@ -122,8 +122,7 @@ def get_firebase_db(path, custom_arguments=None, token=None): def set_firebase_db(path, data, token=None): - config = load_config() - databaseName = config['firebase']['database_name'] + databaseName = CONFIG['firebase']['database_name'] database_url = f'https://{databaseName}.firebaseio.com' request_ref = '{0}{1}.json?auth={2}'.format(database_url, path, token) headers = {"content-type": "application/json; charset=UTF-8"} @@ -141,8 +140,7 @@ def set_firebase_db(path, data, token=None): def update_firebase_db(path, data, token=None): - config = load_config() - databaseName = config['firebase']['database_name'] + databaseName = CONFIG['firebase']['database_name'] database_url = f'https://{databaseName}.firebaseio.com' request_ref = '{0}{1}.json?auth={2}'.format(database_url, path, token) headers = {"content-type": "application/json; charset=UTF-8"}