diff --git a/docker-compose.yaml b/docker-compose.yaml index aea4b2542..ba0c4d263 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -72,7 +72,7 @@ services: build: context: postgres/ environment: - POSTGERS_PASSWORD: '${POSTGRES_PASSWORD}' + POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}' POSTGRES_USER: 'mapswipe_workers' POSTGRES_DB: 'mapswipe' PGDATA: '/var/lib/postgresql/mapswipe' @@ -81,7 +81,7 @@ services: # Set options for WAL-G (backup tool) command: postgres -c archive_mode=on -c archive_timeout=60 -c archive_command="/archive_command.sh %p" volumes: - - ./postgres-data:/var/lib/postgresql/mapswipe + - ./postgres-data:/var/lib/postgresql/mapswipe restart: unless-stopped ports: - "5432:5432" diff --git a/docs/source/backup.md b/docs/source/backup.md index 6f8451048..9a4015713 100644 --- a/docs/source/backup.md +++ b/docs/source/backup.md @@ -14,14 +14,19 @@ The WAL-G backup setup is integrated into the Postgres Docker image. It will do WAL-G is installed alongside Postgres. See the Dockerfile of Postgres (`postgres/Dockerfile`) for details. In the docker-compose postgres command (`docker-compose.yml`) archive parameter of postgres are set needed to make archives. -## Restore setup +### Configuration -The WAL-G restore setup is realized in a dedicated Docker image (`postgres/Dockerfile-restore_backup`). It will create a new Postgres database cluster, fetch latest backup using `wal-g backup-fetch` and create a `recovery.conf` file in the new cluster during Docker build. `recovery.conf` is used by Postgres during first start to get the `restore_command`. Again the exact commands are to be found in `postgres/wal-g/restore_command.sh`. During first start Postgres will get WALs from backup server and restore the database. +To store backups in Google Cloud Storage, WAL-G requires that this variable is set: `WALG_GS_PREFIX` to specify where to store backups (eg. `gs://x4m-test-bucket/walg-folder`). +Please add this to the `.env` file at the root of MapSwipe Back-end (See `.example-env` for environment variables which have to be set) +WAL-G determines Google Cloud credentials using application-default credentials like other GCP tools. Get a Service Account Key file (`serviceAccountKey.json`) for your Google Cloud Storage (See [Google Cloud Docs](https://cloud.google.com/iam/docs/creating-managing-service-account-keys)) and save it to `postgres/serviceAccountKey.json`. -## Configuration -To store backups in Google Cloud Storage, WAL-G requires that this variable be set: `WALG_GS_PREFIX` to specify where to store backups (eg. `gs://x4m-test-bucket/walg-folder`). -Please add this to your `.env` file at the root of MapSwipe Back-end (See `.example-env` for environment variables wich needs to be set) +## Restore setup -WAL-G determines Google Cloud credentials using application-default credentials like other GCP tools. Get a Service Account Key file (`serviceAccountKey.json`) for your Google Cloud Storage (See [Google Cloud Docs](https://cloud.google.com/iam/docs/creating-managing-service-account-keys)) and save it to `postgres/serviceAccountKey.json`. +The WAL-G restore setup is realized in a dedicated Docker image (`postgres/recovery/Dockerfile`). The entrypoint is the `ini_recovery.sh` srcipt. This script will create a new Postgres database cluster, fetch latest backup using `wal-g backup-fetch` and create a `recovery.conf` file in the new cluster. `recovery.conf` is used by Postgres during first start to get the `restore_command`. Again the exact commands are to be found in `postgres/recovery/restore_command.sh`. During first start Postgres will get WALs from backup server and restore the database. + + +### Configuration + +> The same configuration as for the backup setup is requiered. Except the Service Account Key has to be stored at `postgres/recovery/serviceAccountKey`. diff --git a/docs/source/contributing.md b/docs/source/contributing.md index 00d8b498c..5814b692d 100644 --- a/docs/source/contributing.md +++ b/docs/source/contributing.md @@ -1,12 +1,49 @@ # Contributing -This document is a work progress. It should describe how to contribute (code or issues) to the MapSwipe back-end. -To contribute to the MapSwipe back-end please create dedicated feature branches. +## Clone from GitHub ```bash +git clone https://github.com/mapswipe/python-mapswipe-workers.git +cd python-mapswipe-workers +git checkout dev +``` + + +## Install MapSwipe Workers + +Create a Python virtual environment and activate it. Install MapSwipe Workers using pip: + +```bash +python -m venv venv +source venv/bin/activate +pip install --editable . +``` + + +## Feature Branch + +To contribute to the MapSwipe back-end please create dedicated feature branches from dev: + +```bash +git checkout dev git checkout -b featureA git commit -am 'add new project type' git push -u origin featureA -git request-pull origin/master featureA +git request-pull origin/dev featureA +``` + +> Note: If a bug in production (master branch) needs fixing before a new versions of MapSwipe Workers gets released (merging dev into master branch), a hotfix branch should be created. In the hotfix branch the bug should be fixed and then merged back with master and also dev. + + +## Style Guide + +This project uses [black](https://github.com/psf/black) and [flake8](https://gitlab.com/pycqa/flake8) to achieve a unified style. + +Use [pre-commit](https://pre-commit.com/) to run `black` and `flake8` prior to any git commit. `pre-commit`, `black` and `flake8` should already be installed in your virtual environment since they are listed in `requirements.txt`. To setup pre-commit simply run: + ``` +pre-commit install +``` + +From now on `black` and `flake8` should run automatically whenever `git commit` is executed. diff --git a/manager_dashboard/manager_dashboard/create.html b/manager_dashboard/manager_dashboard/create.html index d53d28fa3..46cc1adf2 100644 --- a/manager_dashboard/manager_dashboard/create.html +++ b/manager_dashboard/manager_dashboard/create.html @@ -14,23 +14,31 @@ + - - + + - + + + + + + + + + + - - - + + + + + - - + + - + + + + + + + + + + + + + + + + + - - + + - + + + + + + + + + + + + + + + + + + - - - + + + diff --git a/mapswipe_workers/Dockerfile b/mapswipe_workers/Dockerfile index 820c05674..ff3aa06ce 100644 --- a/mapswipe_workers/Dockerfile +++ b/mapswipe_workers/Dockerfile @@ -4,7 +4,7 @@ FROM thinkwhere/gdal-python:3.7-shippable # Install gdal-bin to get ogr2ogr tool RUN apt-get update -RUN apt-get install gdal-bin +RUN apt-get --yes install gdal-bin # create directories for config, logs and data ARG config_dir=/usr/share/config/mapswipe_workers/ @@ -37,4 +37,4 @@ COPY config $config_dir # RUN python setup.py install RUN pip install . -# we don't use a CMD here, this will be defined in docker-compose.yaml \ No newline at end of file +# we don't use a CMD here, this will be defined in docker-compose.yaml diff --git a/mapswipe_workers/mapswipe_workers/base/base_project.py b/mapswipe_workers/mapswipe_workers/base/base_project.py index 07ea3672a..53bb5da9f 100644 --- a/mapswipe_workers/mapswipe_workers/base/base_project.py +++ b/mapswipe_workers/mapswipe_workers/base/base_project.py @@ -66,7 +66,7 @@ def __init__(self, project_draft): self.projectId = project_draft['projectDraftId'] self.projectType = int(project_draft['projectType']) self.verificationNumber = project_draft['verificationNumber'] - self.status = 'new' + self.status = 'inactive' self.projectTopic = project_draft.get('projectTopic', None) self.projectRegion = project_draft.get('projectRegion', None) self.projectNumber = project_draft.get('projectNumber', None) diff --git a/mapswipe_workers/mapswipe_workers/generate_stats/generate_stats.py b/mapswipe_workers/mapswipe_workers/generate_stats/generate_stats.py index a43a3b001..62214a25d 100644 --- a/mapswipe_workers/mapswipe_workers/generate_stats/generate_stats.py +++ b/mapswipe_workers/mapswipe_workers/generate_stats/generate_stats.py @@ -20,8 +20,8 @@ def generate_stats(project_id_list: list): ---------- project_id_list: list """ - logger.info(f"will generate stats for: {project_id_list}") + logger.info(f"will generate stats for: {project_id_list}") projects_info_filename = f"{DATA_PATH}/api-data/projects/projects_static.csv" projects_df = overall_stats.get_project_static_info(projects_info_filename) project_id_list_postgres = projects_df["project_id"].to_list() @@ -60,10 +60,14 @@ def generate_stats(project_id_list: list): # TODO: for build area projects generate tasking manager geometries - # merge static info and dynamic info and save if len(project_id_list) > 0: + # merge static info and dynamic info and save projects_filename = f"{DATA_PATH}/api-data/projects/projects.csv" - overall_stats.save_projects(projects_filename, projects_df, projects_dynamic_df) + projects_df = overall_stats.save_projects(projects_filename, projects_df, projects_dynamic_df) + + # generate overall stats for active, inactive, finished projects + overall_stats_filename = f"{DATA_PATH}/api-data/stats.csv" + overall_stats.get_overall_stats(projects_df, overall_stats_filename) logger.info(f"finished generate stats for: {project_id_list}") diff --git a/mapswipe_workers/mapswipe_workers/generate_stats/overall_stats.py b/mapswipe_workers/mapswipe_workers/generate_stats/overall_stats.py index 98f0b57af..7ef9b57f8 100644 --- a/mapswipe_workers/mapswipe_workers/generate_stats/overall_stats.py +++ b/mapswipe_workers/mapswipe_workers/generate_stats/overall_stats.py @@ -5,18 +5,57 @@ from mapswipe_workers.utils import geojson_functions -def get_project_static_info(filename): +def get_overall_stats(projects_df: pd.DataFrame, filename: str) -> pd.DataFrame: + """ + The function aggregates the statistics per project using the status attribute. + We derive aggregated statistics for active, inactive and finished projects. + The number of users should not be summed up here, since this would generate wrong results. + A single user can contribute to multiple projects, we need to consider this. + + Parameters + ---------- + projects_df: pd.DataFrame + filename: str + """ + + overall_stats_df = projects_df.groupby(['status']).agg( + count_projects=pd.NamedAgg(column='project_id', aggfunc='count'), + area_sqkm=pd.NamedAgg(column='area_sqkm', aggfunc='sum'), + number_of_results=pd.NamedAgg(column='number_of_results', aggfunc='sum'), + number_of_results_progress=pd.NamedAgg(column='number_of_results_progress', aggfunc='sum'), + average_number_of_users_per_project=pd.NamedAgg(column='number_of_users', aggfunc='mean') + ) + + overall_stats_df.to_csv(filename, index_label="status") + logger.info(f'saved overall stats to {filename}') + + return overall_stats_df + + +def get_project_static_info(filename: str) -> pd.DataFrame: + """ + The function queries the projects table. + Each row represents a single project and provides the information which is static. + By static we understand all attributes which are not affected by new results being contributed. + The results are stored in a csv file and also returned as a pandas DataFrame. + + Parameters + ---------- + filename: str + """ pg_db = auth.postgresDB() + + # make sure to replace newline characters here sql_query = """ COPY ( SELECT project_id - ,name - ,project_details - ,look_for + ,regexp_replace(name, E'[\\n\\r]+', ' ', 'g' ) as name + ,regexp_replace(project_details, E'[\\n\\r]+', ' ', 'g' ) as project_details + ,regexp_replace(look_for, E'[\\n\\r]+', ' ', 'g' ) as look_for ,project_type - ,status + ,regexp_replace(status, E'[\\n\\r]+', ' ', 'g' ) as status ,ST_Area(geom::geography)/1000000 as area_sqkm ,ST_AsText(geom) as geom ,ST_AsText(ST_Centroid(geom)) as centroid @@ -30,10 +69,20 @@ def get_project_static_info(filename): logger.info("got projects from postgres.") df = pd.read_csv(filename) + return df -def load_project_info_dynamic(filename): +def load_project_info_dynamic(filename: str) -> pd.DataFrame: + """ + The function loads data from a csv file into a pandas dataframe. + If not file exists, it will be initialized. + + Parameters + ---------- + filename: str + """ + if os.path.isfile(filename): logger.info(f"file {filename} exists. Init from this file.") df = pd.read_csv(filename, index_col="idx") @@ -52,11 +101,27 @@ def load_project_info_dynamic(filename): return df -def save_projects(filename, df, df_dynamic): +def save_projects(filename: str, df: pd.DataFrame, df_dynamic: pd.DataFrame) -> pd.DataFrame: + """ + The function merges the dataframes for static and dynamic project information + and then save the result as csv file. + Additionally, two geojson files are generated using + (a) the geometry of the projects and + (b) the centroid of the projects. + + Parameters + ---------- + filename: str + df: pd.DataFrame + df_dynamic: pd.DataFrame + """ + projects_df = df.merge( df_dynamic, left_on="project_id", right_on="project_id", how="left" ) - projects_df.to_csv(filename, index_label="idx") + projects_df.to_csv(filename, index_label="idx", line_terminator='\n') logger.info(f"saved projects: {filename}") geojson_functions.csv_to_geojson(filename, "geom") geojson_functions.csv_to_geojson(filename, "centroid") + + return projects_df diff --git a/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_tutorial.py b/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_tutorial.py index 8ec2bd8fa..d3efb4001 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_tutorial.py +++ b/mapswipe_workers/mapswipe_workers/project_types/build_area/build_area_tutorial.py @@ -11,112 +11,119 @@ def create_tutorial(tutorial): def generate_tutorial_data(tutorial): - tutorial_id = tutorial['projectId'] - categories = tutorial['categories'].keys() - logger.info(f'create tutorial for tutorial id: {tutorial_id}') + tutorial_id = tutorial["projectId"] + categories = tutorial["categories"].keys() + logger.info(f"create tutorial for tutorial id: {tutorial_id}") grouped_tasks = { - 'building_easy_1': {}, - 'building_easy_2': {}, - 'maybe_1': {}, - 'no_building_easy_1': {}, - 'no_building_easy_2': {}, - 'bad_clouds_1': {}, - 'bad_no_imagery_1': {}, - 'bad_no_imagery_2': {}, + "building_easy_1": {}, + "building_easy_2": {}, + "maybe_1": {}, + "no_building_easy_1": {}, + "no_building_easy_2": {}, + "bad_clouds_1": {}, + "bad_no_imagery_1": {}, + "bad_no_imagery_2": {}, } c = 0 - with open(tutorial['examplesFile']) as csvDataFile: + with open(tutorial["examplesFile"]) as csvDataFile: logger.info(f'tutorial tasks are based on: {tutorial["examplesFile"]}') csvReader = csv.reader(csvDataFile) for row in csvReader: c += 1 if c == 1: - header = row + pass else: category = row[2] if not row[3] in grouped_tasks[category].keys(): grouped_tasks[category][row[3]] = { - 'task_id_list': [], - 'task_x_list': [], - 'task_y_list': [], - 'reference_answer_list': [], - 'category': category, - 'url_list': [] + "task_id_list": [], + "task_x_list": [], + "task_y_list": [], + "reference_answer_list": [], + "category": category, + "url_list": [], } - grouped_tasks[category][row[3]]['task_id_list'].append(row[0]) - zoom = row[0].split('-')[0] - task_x = int(row[0].split('-')[1]) - task_y = int(row[0].split('-')[2]) - grouped_tasks[category][row[3]]['task_x_list'].append(task_x) - grouped_tasks[category][row[3]]['task_y_list'].append(task_y) - url = t.tile_coords_zoom_and_tileserver_to_URL(task_x, task_y, zoom, tutorial['tileServer']['name'], - auth.get_api_key(tutorial['tileServer']['name']), None, - None) - grouped_tasks[category][row[3]]['url_list'].append(url) - grouped_tasks[category][row[3]]['reference_answer_list'].append(row[1]) + grouped_tasks[category][row[3]]["task_id_list"].append(row[0]) + zoom = row[0].split("-")[0] + task_x = int(row[0].split("-")[1]) + task_y = int(row[0].split("-")[2]) + grouped_tasks[category][row[3]]["task_x_list"].append(task_x) + grouped_tasks[category][row[3]]["task_y_list"].append(task_y) + url = t.tile_coords_zoom_and_tileserver_to_URL( + task_x, + task_y, + zoom, + tutorial["tileServer"]["name"], + auth.get_api_key(tutorial["tileServer"]["name"]), + None, + None, + ) + grouped_tasks[category][row[3]]["url_list"].append(url) + grouped_tasks[category][row[3]]["reference_answer_list"].append(row[1]) groups_dict = {} tasks_dict = {} for i in range(0, 1): - print(i) group_id = 101 + i + # the yMin represents a tile located at the equator at zoom level 18 groups_dict[group_id] = { "xMax": "115", "xMin": "100", - "yMax": "202", - "yMin": "200", + "yMax": "131074", + "yMin": "131072", "requiredCount": 5, "finishedCount": 0, "groupId": group_id, "projectId": tutorial_id, "numberOfTasks": 36, - "progress": 0 + "progress": 0, } tasks_dict[group_id] = {} # select 6 tasks for each category and add to group counter = -1 task_x = 100 - task_y = 200 + # the task_y represents a tile located at the equator at zoom level 18 + task_y = 131072 for category in categories: # print(category) keys = list(grouped_tasks[category].keys()) tasks = grouped_tasks[category][keys[i]] - x_min = min(tasks['task_x_list']) - y_min = min(tasks['task_y_list']) + x_min = min(tasks["task_x_list"]) + y_min = min(tasks["task_y_list"]) # replace taskX and TaskY # TaskY between 0 and 2 # TaskX between 0 and 11 - for j in range(0, len(tasks['task_id_list'])): + for j in range(0, len(tasks["task_id_list"])): counter += 1 task = { - 'taskId_real': tasks['task_id_list'][j], - 'groupId': group_id, - 'projectId': tutorial_id, - 'referenceAnswer': tasks['reference_answer_list'][j], - 'category': tasks['category'], - 'url': tasks['url_list'][j] + "taskId_real": tasks["task_id_list"][j], + "groupId": group_id, + "projectId": tutorial_id, + "referenceAnswer": tasks["reference_answer_list"][j], + "category": tasks["category"], + "url": tasks["url_list"][j], } - if tasks['task_x_list'][j] == x_min: - task['taskX'] = str(task_x) - elif tasks['task_x_list'][j] == (x_min + 1): - task['taskX'] = str(task_x + 1) + if tasks["task_x_list"][j] == x_min: + task["taskX"] = str(task_x) + elif tasks["task_x_list"][j] == (x_min + 1): + task["taskX"] = str(task_x + 1) - if tasks['task_y_list'][j] == y_min: - task['taskY'] = str(task_y) - elif tasks['task_y_list'][j] == (y_min + 1): - task['taskY'] = str(task_y + 1) - elif tasks['task_y_list'][j] == (y_min + 2): - task['taskY'] = str(task_y + 2) + if tasks["task_y_list"][j] == y_min: + task["taskY"] = str(task_y) + elif tasks["task_y_list"][j] == (y_min + 1): + task["taskY"] = str(task_y + 1) + elif tasks["task_y_list"][j] == (y_min + 2): + task["taskY"] = str(task_y + 2) - task['taskId'] = '18-{}-{}'.format(task['taskX'], task['taskY']) + task["taskId"] = "18-{}-{}".format(task["taskX"], task["taskY"]) tasks_dict[group_id][counter] = task task_x += 2 @@ -126,13 +133,15 @@ def generate_tutorial_data(tutorial): def upload_tutorial_to_firebase(tutorial, groups_dict, tasks_dict): # upload groups and tasks to firebase - tutorial_id = tutorial['projectId'] + tutorial_id = tutorial["projectId"] fb_db = auth.firebaseDB() - ref = fb_db.reference('') - ref.update({ - 'v2/projects/{}'.format(tutorial_id): tutorial, - 'v2/groups/{}'.format(tutorial_id): groups_dict, - 'v2/tasks/{}'.format(tutorial_id): tasks_dict, - }) - logger.info(f'uploaded tutorial data to firebase for {tutorial_id}') + ref = fb_db.reference("") + ref.update( + { + "v2/projects/{}".format(tutorial_id): tutorial, + "v2/groups/{}".format(tutorial_id): groups_dict, + "v2/tasks/{}".format(tutorial_id): tasks_dict, + } + ) + logger.info(f"uploaded tutorial data to firebase for {tutorial_id}") diff --git a/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_tutorial.py b/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_tutorial.py index 69ff920cf..32943f52a 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_tutorial.py +++ b/mapswipe_workers/mapswipe_workers/project_types/change_detection/change_detection_tutorial.py @@ -1,6 +1,4 @@ -import csv import json - from mapswipe_workers.definitions import logger from mapswipe_workers import auth from mapswipe_workers.project_types.change_detection import tile_functions as t @@ -12,80 +10,79 @@ def create_tutorial(tutorial): def generate_tutorial_data(tutorial): - tutorial_id = tutorial['projectId'] - logger.info(f'create tutorial for tutorial id: {tutorial_id}') + tutorial_id = tutorial["projectId"] + logger.info(f"create tutorial for tutorial id: {tutorial_id}") groups_dict = {} tasks_dict = {} - with open(tutorial['examplesFile']) as json_file: + with open(tutorial["examplesFile"]) as json_file: tutorial_tasks = json.load(json_file) - print(len(tutorial_tasks['features'])) - #print(tutorial_tasks['features']) - - for feature in tutorial_tasks['features']: - print(feature) + for feature in tutorial_tasks["features"]: + category = feature["properties"]["category"] + group_id = 100 + feature["properties"]["id"] - category = feature['properties']['category'] - group_id = 100 + feature['properties']['id'] + if group_id not in groups_dict.keys(): - if not group_id in groups_dict.keys(): + # yMin 32768 represents a tile located at the equator at zoom level 16 groups_dict[group_id] = { - "xMax": "104", + "xMax": "105", "xMin": "100", - "yMax": "200", - "yMin": "200", + "yMax": "32768", + "yMin": "32768", "requiredCount": 5, "finishedCount": 0, "groupId": group_id, "projectId": tutorial_id, "numberOfTasks": 4, - "progress": 0 + "progress": 0, } - reference = feature['properties']['reference'] - zoom = feature['properties']['TileZ'] - task_x = feature['properties']['TileX'] - task_y = feature['properties']['TileY'] - task_id_real = '{}-{}-{}'.format(zoom, task_x, task_y) + reference = feature["properties"]["reference"] + zoom = feature["properties"]["TileZ"] + task_x = feature["properties"]["TileX"] + task_y = feature["properties"]["TileY"] + task_id_real = "{}-{}-{}".format(zoom, task_x, task_y) urlA = t.tile_coords_zoom_and_tileserver_to_URL( task_x, task_y, zoom, - tutorial['tileServerA']['name'], - tutorial['tileServerA']['apiKey'], - tutorial['tileServerA']['url'], + tutorial["tileServerA"]["name"], + tutorial["tileServerA"]["apiKey"], + tutorial["tileServerA"]["url"], None, ) urlB = t.tile_coords_zoom_and_tileserver_to_URL( task_x, task_y, zoom, - tutorial['tileServerB']['name'], - tutorial['tileServerB']['apiKey'], - tutorial['tileServerB']['url'], + tutorial["tileServerB"]["name"], + tutorial["tileServerB"]["apiKey"], + tutorial["tileServerB"]["url"], None, ) - if not group_id in tasks_dict.keys(): + if group_id not in tasks_dict.keys(): tasks_dict[group_id] = {} - task_id = '{}-{}-{}'.format(16, 100, 200) + task_id = "{}-{}-{}".format(16, 100, 32768) else: - task_id = '{}-{}-{}'.format(16, 100 + len(tasks_dict[group_id].keys()), 200) + task_id = "{}-{}-{}".format( + 16, 100 + len(tasks_dict[group_id].keys()), 32768 + ) task = { - 'taskId_real': task_id_real, - 'taskId': task_id, - 'taskX': 100 + len(tasks_dict[group_id].keys()), - 'taskY': 200, - 'groupId': group_id, - 'projectId': tutorial_id, - 'referenceAnswer': reference, - 'category': category, - 'urlA': urlA, - 'urlB': urlB, + "taskId_real": task_id_real, + "taskId": task_id, + "taskX": 100 + len(tasks_dict[group_id].keys()), + "taskY": 32768, + "groupId": group_id, + "projectId": tutorial_id, + "referenceAnswer": reference, + "category": category, + "urlA": urlA, + "urlB": urlB, } tasks_dict[group_id][len(tasks_dict[group_id].keys())] = task @@ -95,13 +92,15 @@ def generate_tutorial_data(tutorial): def upload_tutorial_to_firebase(tutorial, groups_dict, tasks_dict): # upload groups and tasks to firebase - tutorial_id = tutorial['projectId'] + tutorial_id = tutorial["projectId"] fb_db = auth.firebaseDB() - ref = fb_db.reference('') - ref.update({ - 'v2/projects/{}'.format(tutorial_id): tutorial, - 'v2/groups/{}'.format(tutorial_id): groups_dict, - 'v2/tasks/{}'.format(tutorial_id): tasks_dict, - }) - logger.info(f'uploaded tutorial data to firebase for {tutorial_id}') + ref = fb_db.reference("") + ref.update( + { + "v2/projects/{}".format(tutorial_id): tutorial, + "v2/groups/{}".format(tutorial_id): groups_dict, + "v2/tasks/{}".format(tutorial_id): tasks_dict, + } + ) + logger.info(f"uploaded tutorial data to firebase for {tutorial_id}") diff --git a/postgres/initdb.sql b/postgres/initdb.sql index 589f6cb8c..991f623b6 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -1,15 +1,11 @@ -- noinspection SqlNoDataSourceInspectionForFile - - CREATE EXTENSION postgis; --- --- TABLES --- + CREATE TABLE IF NOT EXISTS projects ( archive boolean, created timestamp, created_by varchar, - geom geometry(MULTIPOLYGON,4326), + geom geometry(MULTIPOLYGON, 4326), image varchar, is_featured boolean, look_for varchar, @@ -23,8 +19,8 @@ CREATE TABLE IF NOT EXISTS projects ( status varchar, verification_number int, project_type_specifics json, - PRIMARY KEY(project_id) - ); + PRIMARY KEY (project_id) +); CREATE TABLE IF NOT EXISTS groups ( project_id varchar, @@ -34,34 +30,42 @@ CREATE TABLE IF NOT EXISTS groups ( required_count int, progress int, project_type_specifics json, - PRIMARY KEY(project_id, group_id), + PRIMARY KEY (project_id, group_id), FOREIGN KEY (project_id) REFERENCES projects (project_id) - ); +); + +CREATE INDEX IF NOT EXISTS groups_projectid ON public.groups + USING btree (group_id); -CREATE INDEX IF NOT EXISTS groups_projectid ON public.groups USING btree (group_id); -CREATE INDEX IF NOT EXISTS groups_goupid ON public.groups USING btree (project_id); +CREATE INDEX IF NOT EXISTS groups_goupid ON public.groups + USING btree (project_id); CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(MULTIPOLYGON,4326), + geom geometry(MULTIPOLYGON, 4326), project_type_specifics json, - PRIMARY KEY(project_id, group_id, task_id), + PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), - FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id) - ); + FOREIGN KEY (project_id, group_id) REFERENCES GROUPS (project_id, group_id) +); -CREATE INDEX IF NOT EXISTS tasks_task_id ON public.tasks USING btree (task_id); -CREATE INDEX IF NOT EXISTS tasks_groupid ON public.tasks USING btree (group_id); -CREATE INDEX IF NOT EXISTS tasks_projectid ON public.tasks USING btree (project_id); +CREATE INDEX IF NOT EXISTS tasks_task_id ON public.tasks + USING btree (task_id); + +CREATE INDEX IF NOT EXISTS tasks_groupid ON public.tasks + USING btree (group_id); + +CREATE INDEX IF NOT EXISTS tasks_projectid ON public.tasks + USING btree (project_id); CREATE TABLE IF NOT EXISTS users ( user_id varchar, username varchar, created timestamp, - PRIMARY KEY(user_id) - ); + PRIMARY KEY (user_id) +); CREATE TABLE IF NOT EXISTS results ( project_id varchar, @@ -74,15 +78,22 @@ CREATE TABLE IF NOT EXISTS results ( result int, PRIMARY KEY (project_id, group_id, task_id, user_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), - FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id), + FOREIGN KEY (project_id, group_id) REFERENCES GROUPS (project_id, group_id), FOREIGN KEY (project_id, group_id, task_id) REFERENCES tasks (project_id, group_id, task_id), FOREIGN KEY (user_id) REFERENCES users (user_id) - ); +); + +CREATE INDEX IF NOT EXISTS results_projectid ON public.results + USING btree (project_id); -CREATE INDEX IF NOT EXISTS results_projectid ON public.results USING btree (project_id); -CREATE INDEX IF NOT EXISTS results_groupid ON public.results USING btree (group_id); -CREATE INDEX IF NOT EXISTS results_taskid ON public.results USING btree (task_id); -CREATE INDEX IF NOT EXISTS results_userid ON public.results USING btree (user_id); +CREATE INDEX IF NOT EXISTS results_groupid ON public.results + USING btree (group_id); + +CREATE INDEX IF NOT EXISTS results_taskid ON public.results + USING btree (task_id); + +CREATE INDEX IF NOT EXISTS results_userid ON public.results + USING btree (user_id); -- create table for results import through csv CREATE TABLE IF NOT EXISTS results_temp ( @@ -94,15 +105,5 @@ CREATE TABLE IF NOT EXISTS results_temp ( start_time timestamp, end_time timestamp, result int, - PRIMARY KEY (project_id, group_id, task_id, user_id), - FOREIGN KEY (project_id) REFERENCES projects (project_id), - FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id), - FOREIGN KEY (project_id, group_id, task_id) REFERENCES tasks (project_id, group_id, task_id), - FOREIGN KEY (user_id) REFERENCES users (user_id) - ); +); --- --- VIEWS --- --- create views for statistics -\i /docker-entrypoint-initdb.d/stat_views.sql diff --git a/postgres/triggers.sql b/postgres/triggers.sql deleted file mode 100644 index 7564cf605..000000000 --- a/postgres/triggers.sql +++ /dev/null @@ -1,61 +0,0 @@ -CREATE TABLE - row_counts ( - relname text PRIMARY KEY, - reltuples numeric - ); - -CREATE OR REPLACE FUNCTION count_trig() -RETURNS TRIGGER AS -$$ - DECLARE - BEGIN - IF TG_OP = 'INSERT' THEN - EXECUTE 'UPDATE row_counts set reltuples=reltuples +1 where relname = ''' || TG_RELNAME || ''''; - RETURN NEW; - ELSIF TG_OP = 'DELETE' THEN - EXECUTE 'UPDATE row_counts set reltuples=reltuples -1 where relname = ''' || TG_RELNAME || ''''; - RETURN OLD; - END IF; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION add_count_trigs() -RETURNS void AS -$$ - DECLARE - rec RECORD; - q text; - BEGIN - FOR rec IN SELECT relname - FROM pg_class r JOIN pg_namespace n ON (relnamespace = n.oid) - WHERE relkind = 'r' AND n.nspname = 'public' LOOP - q := 'CREATE TRIGGER ' || rec.relname || '_count BEFORE INSERT OR DELETE ON ' ; - q := q || rec.relname || ' FOR EACH ROW EXECUTE PROCEDURE count_trig()'; - EXECUTE q; - END LOOP; - RETURN; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION init_row_counts() -RETURNS void AS -$$ - DECLARE - rec RECORD; - crec RECORD; - BEGIN - FOR rec IN SELECT relname - FROM pg_class r JOIN pg_namespace n ON (relnamespace = n.oid) - WHERE relkind = 'r' AND n.nspname = 'public' LOOP - FOR crec IN EXECUTE 'SELECT count(*) as rows from '|| rec.relname LOOP - -- nothing here, move along - END LOOP; - INSERT INTO row_counts values (rec.relname, crec.rows) ; - END LOOP; - - RETURN; - END; -$$ -LANGUAGE 'plpgsql';