From 1782f3193a432ce3a0cd14c6590cbc35f69c8440 Mon Sep 17 00:00:00 2001 From: h9b Date: Tue, 28 Apr 2020 16:35:37 +0200 Subject: [PATCH 1/6] tm geometries as bounding box, 7 digits #333 --- .../utils/geojson_functions.py | 77 ++++++++++++++++--- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py b/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py index 34aeac155..e9f00c042 100644 --- a/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py +++ b/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py @@ -27,7 +27,7 @@ def csv_to_geojson(filename: str, geometry_field: str = "geom"): outfile, filename, "-sql", - f'SELECT *, CAST({geometry_field} as geometry) FROM "{filename_without_path}"', + f'SELECT *, CAST({geometry_field} as geometry) FROM "{filename_without_path}"', # noqa E501 ], check=True, ) @@ -101,28 +101,59 @@ def add_metadata_to_geojson(filename: str, geometry_field: str = "geom"): logger.info(f"added metadata to {filename}.") -def create_group_geom(group_data): +def create_group_geom(group_data, shape="bounding_box"): + """Create bounding box or convex hull of input task geometries.""" + result_geom_collection = ogr.Geometry(ogr.wkbMultiPolygon) for result, data in group_data.items(): result_geom = ogr.CreateGeometryFromWkt(data["wkt"]) result_geom_collection.AddGeometry(result_geom) - group_geom = result_geom_collection.ConvexHull() + if shape == "convex_hull": + group_geom = result_geom_collection.ConvexHull() + elif shape == "bounding_box": + # Get Envelope + lon_left, lon_right, lat_top, lat_bottom = result_geom_collection.GetEnvelope() + + # Create Geometry + ring = ogr.Geometry(ogr.wkbLinearRing) + ring.AddPoint(lon_left, lat_top) + ring.AddPoint(lon_right, lat_top) + ring.AddPoint(lon_right, lat_bottom) + ring.AddPoint(lon_left, lat_bottom) + ring.AddPoint(lon_left, lat_top) + group_geom = ogr.Geometry(ogr.wkbPolygon) + group_geom.AddGeometry(ring) + return group_geom def create_geojson_file_from_dict(final_groups_dict, outfile): - # TODO: adapt input name + """Take output from generate stats and create TM geometries. + + In order to create a GeoJSON file with a coordinate precision of 7 + we take a small detour. + First, we create a GeoJSONSeq file. + This contains only the features. + Then we add these features to the final GeoJSON file. + The current shape of the output geometries is set to 'bounding_box'. + """ - driver = ogr.GetDriverByName("GeoJSON") + driver = ogr.GetDriverByName("GeoJSONSeq") # define spatial Reference srs = osr.SpatialReference() srs.ImportFromEPSG(4326) + outfile_temp = outfile.replace(".geojson", "_temp.geojson") + + if os.path.exists(outfile_temp): + driver.DeleteDataSource(outfile_temp) + if os.path.exists(outfile): driver.DeleteDataSource(outfile) - dataSource = driver.CreateDataSource(outfile) + + dataSource = driver.CreateDataSource(outfile_temp) # create layer - layer = dataSource.CreateLayer(outfile, srs, geom_type=ogr.wkbPolygon) + layer = dataSource.CreateLayer(outfile_temp, srs, geom_type=ogr.wkbPolygon,) # create fields field_id = ogr.FieldDefn("group_id", ogr.OFTInteger) @@ -133,7 +164,8 @@ def create_geojson_file_from_dict(final_groups_dict, outfile): else: for group_id in final_groups_dict.keys(): group_data = final_groups_dict[group_id] - group_geom = create_group_geom(group_data) + # create the final group geometry + group_geom = create_group_geom(group_data, "bounding_box") final_groups_dict[group_id]["group_geom"] = group_geom # init feature @@ -163,8 +195,35 @@ def create_geojson_file_from_dict(final_groups_dict, outfile): print(group_geom) continue + # make sure to close layer and data source layer = None - logger.info("created outfile: %s." % outfile) + dataSource = None + + # load the features from temp file + feature_collection = [] + with open(outfile_temp, "r") as f: + for cnt, line in enumerate(f): + feature_collection.append(json.loads(line)) + + # create final geojson structure + geojson_structure = { + "type": "FeatureCollection", + "name": outfile, + "crs": { + "type": "name", + "properties": {"name": "urn:ogc:def:crs:OGC:1.3:CRS84"}, + }, + "features": feature_collection, + } + + # save to geojson + with open(outfile, "w") as json_file: + json.dump(geojson_structure, json_file) + logger.info("created outfile: %s." % outfile) + + # remove temp file + if os.path.exists(outfile_temp): + driver.DeleteDataSource(outfile_temp) def create_geojson_file(geometries, outfile): From bd88c0c0e15028dfb738f4769418381c9f0bcf1d Mon Sep 17 00:00:00 2001 From: h9b Date: Tue, 28 Apr 2020 19:31:38 +0200 Subject: [PATCH 2/6] fix wrong logger name, add project geom file to api during project creation. #333 --- .../mapswipe_workers/base/base_project.py | 494 +++++++++--------- .../mapswipe_workers/definitions.py | 2 +- .../mapswipe_workers/mapswipe_workers.py | 2 +- .../utils/geojson_functions.py | 40 +- 4 files changed, 296 insertions(+), 242 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/base/base_project.py b/mapswipe_workers/mapswipe_workers/base/base_project.py index 28928e06f..29ab9ab01 100644 --- a/mapswipe_workers/mapswipe_workers/base/base_project.py +++ b/mapswipe_workers/mapswipe_workers/base/base_project.py @@ -4,11 +4,13 @@ import json import os import sys +import ogr from mapswipe_workers import auth from mapswipe_workers.definitions import DATA_PATH from mapswipe_workers.definitions import logger from mapswipe_workers.definitions import CustomError +from mapswipe_workers.utils import geojson_functions class BaseProject(metaclass=ABCMeta): @@ -51,25 +53,25 @@ def __init__(self, project_draft): """ self.created = dt.datetime.now() - self.createdBy = project_draft['createdBy'] + self.createdBy = project_draft["createdBy"] self.groups = list() - self.groupMaxSize = project_draft.get('groupMaxSize', 0) + self.groupMaxSize = project_draft.get("groupMaxSize", 0) self.resultCount = 0 - self.image = project_draft['image'] + self.image = project_draft["image"] self.isFeatured = False - self.lookFor = project_draft['lookFor'] - self.name = project_draft['name'] + self.lookFor = project_draft["lookFor"] + self.name = project_draft["name"] self.requiredResults = 0 self.progress = 0 - self.projectDetails = project_draft['projectDetails'] - self.projectId = project_draft['projectDraftId'] - self.projectType = int(project_draft['projectType']) - self.verificationNumber = project_draft['verificationNumber'] - self.status = 'inactive' - self.projectTopic = project_draft.get('projectTopic', None) - self.projectRegion = project_draft.get('projectRegion', None) - self.projectNumber = project_draft.get('projectNumber', None) - self.requestingOrganisation = project_draft.get('requestingOrganisation', None) + self.projectDetails = project_draft["projectDetails"] + self.projectId = project_draft["projectDraftId"] + self.projectType = int(project_draft["projectType"]) + self.verificationNumber = project_draft["verificationNumber"] + self.status = "inactive" + self.projectTopic = project_draft.get("projectTopic", None) + self.projectRegion = project_draft.get("projectRegion", None) + self.projectNumber = project_draft.get("projectNumber", None) + self.requestingOrganisation = project_draft.get("requestingOrganisation", None) # TODO: Implement resultRequiredCounter as property. # Does not work because for some reason project['group'] = vars() @@ -78,6 +80,7 @@ def __init__(self, project_draft): # def resultRequiredCounter(self): # return self.resultRequiredCounter + @property def save_project(self): """ Creates a projects with groups and tasks @@ -87,10 +90,7 @@ def save_project(self): ------ Boolean: True = Successful """ - logger.info( - f'{self.projectId}' - f' - start creating a project' - ) + logger.info(f"{self.projectId}" f" - start creating a project") # Convert object attributes to dictonaries # for saving it to firebase and postgres @@ -100,17 +100,17 @@ def save_project(self): for group in self.groups: group = vars(group) tasks = list() - for task in group['tasks']: + for task in group["tasks"]: tasks.append(vars(task)) - groupsOfTasks[group['groupId']] = tasks - del group['tasks'] - groups[group['groupId']] = group - del(project['groups']) - project.pop('inputGeometries', None) - project.pop('validInputGeometries', None) + groupsOfTasks[group["groupId"]] = tasks + del group["tasks"] + groups[group["groupId"]] = group + del project["groups"] + project.pop("inputGeometries", None) + project.pop("validInputGeometries", None) # Convert Date object to ISO Datetime: # https://www.w3.org/TR/NOTE-datetime - project['created'] = self.created.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + project["created"] = self.created.strftime("%Y-%m-%dT%H:%M:%S.%fZ") # logger.info( # f'{self.projectId}' # f' - size of all tasks: ' @@ -119,141 +119,148 @@ def save_project(self): # Make sure projects get saved in Postgres and Firebase successful try: self.save_to_postgres( - project, - groups, - groupsOfTasks, - ) + project, groups, groupsOfTasks, + ) logger.info( - f'{self.projectId}' - f' - the project has been saved' - f' to postgres' - ) - try: - self.save_to_firebase( - project, - groups, - groupsOfTasks, - ) - logger.info( - f'{self.projectId}' - f' - the project has been saved' - f' to firebase' - ) - return True - except Exception as e: - self.delete_from_postgres() - logger.exception( - f'{self.projectId}' - f' - the project could not be saved' - f' to firebase. ' - ) + f"{self.projectId}" f" - the project has been saved" f" to postgres" + ) + except Exception as e: + logger.exception( + f"{self.projectId}" + f" - the project could not be saved" + f" to postgres and will therefor not be " + f" saved to firebase" + ) + raise CustomError(e) - logger.info(f'{self.projectId} deleted project data from postgres') - raise CustomError(e) + # if project can't be saved to files, delete also in postgres + try: + self.save_to_files(project) + logger.info( + f"{self.projectId}" f" - the project has been saved" f" to files" + ) except Exception as e: + self.delete_from_postgres() logger.exception( - f'{self.projectId}' - f' - the project could not be saved' - f' to postgres and will therefor not be ' - f' saved to firebase' - ) + f"{self.projectId}" f" - the project could not be saved" f" to files. " + ) + logger.info( + f"{self.projectId} deleted project data from files and postgres" + ) raise CustomError(e) + try: + self.save_to_firebase( + project, groups, groupsOfTasks, + ) + logger.info( + f"{self.projectId}" f" - the project has been saved" f" to firebase" + ) + # if project can't be saved to firebase, delete also in postgres + except Exception as e: + self.delete_from_postgres() + self.delete_from_files() + logger.exception( + f"{self.projectId}" + f" - the project could not be saved" + f" to firebase. " + ) + + logger.info( + f"{self.projectId} deleted project data from postgres and files" + ) + raise CustomError(e) + + return True + def save_to_firebase(self, project, groups, groupsOfTasks): # remove wkt geometry attribute of projects and tasks - project.pop('geometry', None) + project.pop("geometry", None) for group_id in groupsOfTasks.keys(): for i in range(0, len(groupsOfTasks[group_id])): - groupsOfTasks[group_id][i].pop('geometry', None) - + groupsOfTasks[group_id][i].pop("geometry", None) fb_db = auth.firebaseDB() - ref = fb_db.reference('') + ref = fb_db.reference("") # save project - ref.update({ - f'v2/projects/{self.projectId}': project - }) + ref.update({f"v2/projects/{self.projectId}": project}) logger.info( - f'{self.projectId} -' - f' uploaded project to firebase realtime database' - ) + f"{self.projectId} -" f" uploaded project to firebase realtime database" + ) # save groups - ref.update({ - f'v2/groups/{self.projectId}': groups - }) + ref.update({f"v2/groups/{self.projectId}": groups}) logger.info( - f'{self.projectId} -' - f' uploaded groups to firebase realtime database' + f"{self.projectId} -" f" uploaded groups to firebase realtime database" ) # save tasks, to avoid firebase write size limit we write chunks of task # we write the tasks for 250 groups at once task_upload_dict = {} - logger.info(f'there are {len(groupsOfTasks)} groups for this project') + logger.info(f"there are {len(groupsOfTasks)} groups for this project") c = 0 for group_id, tasks_list in groupsOfTasks.items(): c += 1 - task_upload_dict[f'v2/tasks/{self.projectId}/{group_id}'] = tasks_list + task_upload_dict[f"v2/tasks/{self.projectId}/{group_id}"] = tasks_list if len(task_upload_dict) % 150 == 0 or c == len(groupsOfTasks): ref.update(task_upload_dict) logger.info( - f'{self.projectId} -' - f' uploaded 150 groups with tasks to firebase realtime database' + f"{self.projectId} -" + f" uploaded 150 groups with tasks to firebase realtime database" ) task_upload_dict = {} - ref = fb_db.reference(f'v2/projectDrafts/{self.projectId}') + ref = fb_db.reference(f"v2/projectDrafts/{self.projectId}") ref.set({}) def save_to_postgres(self, project, groups, groupsOfTasks): - ''' + """ Defines SQL queries and data for import a project into postgres. SQL queries will be executed as transaction. (Either every query will be executed or none) - ''' + """ - query_insert_project = ''' + query_insert_project = """ INSERT INTO projects VALUES (%s,%s,ST_Force2D(ST_Multi(ST_GeomFromText(%s, 4326))),%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s); - ''' + """ data_project = [ - self.created, - self.createdBy, - project['geometry'], - project['image'], - project['isFeatured'], - project['lookFor'], - project['name'], - project['progress'], - project['projectDetails'], - project['projectId'], - project['projectType'], - project['requiredResults'], - project['resultCount'], - project['status'], - project['verificationNumber'] + self.created, + self.createdBy, + project["geometry"], + project["image"], + project["isFeatured"], + project["lookFor"], + project["name"], + project["progress"], + project["projectDetails"], + project["projectId"], + project["projectType"], + project["requiredResults"], + project["resultCount"], + project["status"], + project["verificationNumber"], ] project_attributes = [ - 'created', - 'createdBy', - 'geometry', - 'image', - 'isFeatured', - 'lookFor', - 'name', - 'progress', - 'projectDetails', - 'projectId', - 'projectType', - 'requiredResults', - 'resultCount', - 'status', - 'verificationNumber' - ] + "created", + "createdBy", + "geometry", + "image", + "isFeatured", + "lookFor", + "name", + "progress", + "projectDetails", + "projectId", + "projectType", + "requiredResults", + "resultCount", + "status", + "verificationNumber", + ] project_type_specifics = dict() for key, value in project.items(): @@ -261,7 +268,7 @@ def save_to_postgres(self, project, groups, groupsOfTasks): project_type_specifics[key] = value data_project.append(json.dumps(project_type_specifics)) - query_recreate_raw_groups = ''' + query_recreate_raw_groups = """ DROP TABLE IF EXISTS raw_groups CASCADE; CREATE TABLE raw_groups ( project_id varchar, @@ -272,9 +279,9 @@ def save_to_postgres(self, project, groups, groupsOfTasks): progress int, project_type_specifics json ); - ''' + """ - query_insert_raw_groups = ''' + query_insert_raw_groups = """ INSERT INTO groups SELECT project_id, @@ -286,9 +293,9 @@ def save_to_postgres(self, project, groups, groupsOfTasks): project_type_specifics FROM raw_groups; DROP TABLE IF EXISTS raw_groups CASCADE; - ''' + """ - query_recreate_raw_tasks = ''' + query_recreate_raw_tasks = """ DROP TABLE IF EXISTS raw_tasks CASCADE; CREATE TABLE raw_tasks ( project_id varchar, @@ -297,9 +304,9 @@ def save_to_postgres(self, project, groups, groupsOfTasks): geom varchar, project_type_specifics json ); - ''' + """ - query_insert_raw_tasks = ''' + query_insert_raw_tasks = """ INSERT INTO tasks SELECT project_id, @@ -309,27 +316,28 @@ def save_to_postgres(self, project, groups, groupsOfTasks): project_type_specifics FROM raw_tasks; DROP TABLE IF EXISTS raw_tasks CASCADE; - ''' + """ groups_txt_filename = self.create_groups_txt_file(groups) tasks_txt_filename = self.create_tasks_txt_file(groupsOfTasks) groups_columns = [ - 'project_id', - 'group_id', - 'number_of_tasks', - 'finished_count', - 'required_count', - 'progress', - 'project_type_specifics' - ] + "project_id", + "group_id", + "number_of_tasks", + "finished_count", + "required_count", + "progress", + "project_type_specifics", + ] tasks_columns = [ - 'project_id', - 'group_id', - 'task_id', - 'geom', - 'project_type_specifics'] + "project_id", + "group_id", + "task_id", + "geom", + "project_type_specifics", + ] # execution of all SQL-Statements as transaction # (either every query gets executed or none) @@ -339,18 +347,12 @@ def save_to_postgres(self, project, groups, groupsOfTasks): p_con._db_cur.execute(query_insert_project, data_project) p_con._db_cur.execute(query_recreate_raw_groups, None) p_con._db_cur.execute(query_recreate_raw_tasks, None) - with open(groups_txt_filename, 'r') as groups_file: - p_con._db_cur.copy_from( - groups_file, - 'raw_groups', - columns=groups_columns - ) - with open(tasks_txt_filename, 'r') as tasks_file: + with open(groups_txt_filename, "r") as groups_file: p_con._db_cur.copy_from( - tasks_file, - 'raw_tasks', - columns=tasks_columns - ) + groups_file, "raw_groups", columns=groups_columns + ) + with open(tasks_txt_filename, "r") as tasks_file: + p_con._db_cur.copy_from(tasks_file, "raw_tasks", columns=tasks_columns) p_con._db_cur.execute(query_insert_raw_groups, None) p_con._db_cur.execute(query_insert_raw_tasks, None) p_con._db_connection.commit() @@ -362,6 +364,30 @@ def save_to_postgres(self, project, groups, groupsOfTasks): os.remove(groups_txt_filename) os.remove(tasks_txt_filename) + def save_to_files(self, project): + """Save the project extent geometry as a GeoJSON file.""" + if not os.path.isdir("{}/api/project_geometries".format(DATA_PATH)): + os.mkdir("{}/api/project_geometries".format(DATA_PATH)) + + outfile = ( + f"{DATA_PATH}/api/project_geometries/project_geom_{self.projectId}.geojson" + ) + print(project) + + wkt_geom = project["geometry"] + geometries = [ogr.CreateGeometryFromWkt(wkt_geom)] + geojson_functions.create_geojson_file(geometries, outfile) + + def delete_from_files(self): + """Delete the project extent geometry file.""" + outfile = ( + f"{DATA_PATH}/project_geometries/project_geom_{self.projectId}.geojson" + ) + try: + os.remove(outfile) + except FileNotFoundError: + pass + def create_groups_txt_file(self, groups): """ Creates a text file containing groups information @@ -381,58 +407,53 @@ def create_groups_txt_file(self, groups): Filename """ - if not os.path.isdir('{}/tmp'.format(DATA_PATH)): - os.mkdir('{}/tmp'.format(DATA_PATH)) + if not os.path.isdir("{}/tmp".format(DATA_PATH)): + os.mkdir("{}/tmp".format(DATA_PATH)) # create txt file with header for later # import with copy function into postgres - groups_txt_filename = ( - f'{DATA_PATH}/tmp/raw_groups_{self.projectId}.txt' - ) - groups_txt_file = open(groups_txt_filename, 'w', newline='') + groups_txt_filename = f"{DATA_PATH}/tmp/raw_groups_{self.projectId}.txt" + groups_txt_file = open(groups_txt_filename, "w", newline="") fieldnames = ( - 'project_id', - 'group_id', - 'number_of_tasks', - 'finished_count', - 'required_count', - 'progress', - 'project_type_specifics' - ) + "project_id", + "group_id", + "number_of_tasks", + "finished_count", + "required_count", + "progress", + "project_type_specifics", + ) w = csv.DictWriter( - groups_txt_file, - fieldnames=fieldnames, - delimiter='\t', - quotechar="'", - ) + groups_txt_file, fieldnames=fieldnames, delimiter="\t", quotechar="'", + ) for groupId, group in groups.items(): try: output_dict = { "project_id": self.projectId, "group_id": groupId, - "number_of_tasks": group['numberOfTasks'], - "finished_count": group['finishedCount'], - "required_count": group['requiredCount'], - "progress": group['progress'], - "project_type_specifics": dict() + "number_of_tasks": group["numberOfTasks"], + "finished_count": group["finishedCount"], + "required_count": group["requiredCount"], + "progress": group["progress"], + "project_type_specifics": dict(), } for key in group.keys(): if key not in output_dict.keys(): - output_dict['project_type_specifics'][key] = group[key] - output_dict['project_type_specifics'] = json.dumps( - output_dict['project_type_specifics'] - ) + output_dict["project_type_specifics"][key] = group[key] + output_dict["project_type_specifics"] = json.dumps( + output_dict["project_type_specifics"] + ) w.writerow(output_dict) except Exception as e: logger.exception( - f'{self.projectId}' - f' - set_groups_postgres - ' - f'groups missed critical information: {e}' - ) + f"{self.projectId}" + f" - set_groups_postgres - " + f"groups missed critical information: {e}" + ) groups_txt_file.close() @@ -458,42 +479,39 @@ def create_tasks_txt_file(self, groupsOfTasks): Filename """ - if not os.path.isdir('{}/tmp'.format(DATA_PATH)): - os.mkdir('{}/tmp'.format(DATA_PATH)) + if not os.path.isdir("{}/tmp".format(DATA_PATH)): + os.mkdir("{}/tmp".format(DATA_PATH)) # save tasks in txt file - tasks_txt_filename = f'{DATA_PATH}/tmp/raw_tasks_{self.projectId}.txt' - tasks_txt_file = open(tasks_txt_filename, 'w', newline='') + tasks_txt_filename = f"{DATA_PATH}/tmp/raw_tasks_{self.projectId}.txt" + tasks_txt_file = open(tasks_txt_filename, "w", newline="") fieldnames = ( - 'project_id', - 'group_id', - 'task_id', - 'geom', - 'project_type_specifics' - ) + "project_id", + "group_id", + "task_id", + "geom", + "project_type_specifics", + ) w = csv.DictWriter( - tasks_txt_file, - fieldnames=fieldnames, - delimiter='\t', - quotechar="'", - ) + tasks_txt_file, fieldnames=fieldnames, delimiter="\t", quotechar="'", + ) for groupId, tasks in groupsOfTasks.items(): for task in tasks: output_dict = { - "project_id": self.projectId, - "group_id": groupId, - "task_id": task['taskId'], - "geom": task['geometry'], - "project_type_specifics": dict() - } + "project_id": self.projectId, + "group_id": groupId, + "task_id": task["taskId"], + "geom": task["geometry"], + "project_type_specifics": dict(), + } for key in task.keys(): if key not in output_dict.keys(): - output_dict['project_type_specifics'][key] = task[key] - output_dict['project_type_specifics'] = json.dumps( - output_dict['project_type_specifics'] - ) + output_dict["project_type_specifics"][key] = task[key] + output_dict["project_type_specifics"] = json.dumps( + output_dict["project_type_specifics"] + ) w.writerow(output_dict) tasks_txt_file.close() @@ -502,11 +520,11 @@ def create_tasks_txt_file(self, groupsOfTasks): def delete_from_postgres(self): p_con = auth.postgresDB() - sql_query = ''' + sql_query = """ DELETE FROM tasks WHERE project_id = %s; DELETE FROM groups WHERE project_id = %s; DELETE FROM projects WHERE project_id = %s; - ''' + """ data = [ self.projectId, self.projectId, @@ -515,57 +533,59 @@ def delete_from_postgres(self): p_con.query(sql_query, data) del p_con logger.info( - f'{self.projectId} - ' - f'deleted project, groups and tasks ' - f'from postgres' - ) + f"{self.projectId} - " + f"deleted project, groups and tasks " + f"from postgres" + ) def calc_required_results(self): for group in self.groups: group.requiredCount = self.verificationNumber self.requiredResults = ( - self.requiredResults + - group.requiredCount * - group.numberOfTasks - ) + self.requiredResults + group.requiredCount * group.numberOfTasks + ) def get_tile_server(self, tile_server): - ''' + """ Creates a dictonary with informations of the tile server with project draft values or default values. - ''' + """ - name = tile_server.get('name', 'bing') + name = tile_server.get("name", "bing") - url = tile_server.get('url', auth.get_tileserver_url(tile_server.get('name', 'bing'))) - if url == '': - url = auth.get_tileserver_url(tile_server.get('name', 'bing')) + url = tile_server.get( + "url", auth.get_tileserver_url(tile_server.get("name", "bing")) + ) + if url == "": + url = auth.get_tileserver_url(tile_server.get("name", "bing")) - apiKeyRequired = tile_server.get('apiKeyRequired') + apiKeyRequired = tile_server.get("apiKeyRequired") - apiKey = tile_server.get('apiKey', auth.get_api_key(tile_server.get('name', 'bing'))) - if apiKey == '': - apiKey = auth.get_api_key(tile_server.get('name', 'bing')) + apiKey = tile_server.get( + "apiKey", auth.get_api_key(tile_server.get("name", "bing")) + ) + if apiKey == "": + apiKey = auth.get_api_key(tile_server.get("name", "bing")) - wmtsLayerName = tile_server.get('wmtsLayerName', None) - if wmtsLayerName == '': + wmtsLayerName = tile_server.get("wmtsLayerName", None) + if wmtsLayerName == "": wmtsLayerName = None - captions = tile_server.get('caption', None) + captions = tile_server.get("caption", None) - date = tile_server.get('date', None) + date = tile_server.get("date", None) - credits = tile_server.get('credits', '') + credits = tile_server.get("credits", "") tile_server_dict = { - 'name': name, - 'url': url, - 'apiKeyRequired': apiKeyRequired, - 'apiKey': apiKey, - 'wmtsLayerName': wmtsLayerName, - 'captions': captions, - 'date': date, - 'credits': credits + "name": name, + "url": url, + "apiKeyRequired": apiKeyRequired, + "apiKey": apiKey, + "wmtsLayerName": wmtsLayerName, + "captions": captions, + "date": date, + "credits": credits, } return tile_server_dict diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index 6d477e437..a823b50c7 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -52,7 +52,7 @@ class CustomError(Exception): } logging.config.dictConfig(LOGGING_CONFIG) -logger = logging.getLogger("Mapswipe Workers") +logger = logging.getLogger("mapswipe") try: diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index bb96ee81e..2e81e0f26 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -83,7 +83,7 @@ def run_create_projects(): project.create_groups() project.calc_required_results() # Save project and its groups and tasks to Firebase and Postgres. - project.save_project() + project.save_project send_slack_message("success", project_name, project.projectId) logger.info("Success: Project Creation ({0})".format(project_name)) except CustomError: diff --git a/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py b/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py index e9f00c042..c8370840b 100644 --- a/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py +++ b/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py @@ -228,15 +228,21 @@ def create_geojson_file_from_dict(final_groups_dict, outfile): def create_geojson_file(geometries, outfile): - driver = ogr.GetDriverByName("GeoJSON") + driver = ogr.GetDriverByName("GeoJSONSeq") # define spatial Reference srs = osr.SpatialReference() srs.ImportFromEPSG(4326) + outfile_temp = outfile.replace(".geojson", "_temp.geojson") + + if os.path.exists(outfile_temp): + driver.DeleteDataSource(outfile_temp) + if os.path.exists(outfile): driver.DeleteDataSource(outfile) - dataSource = driver.CreateDataSource(outfile) + + dataSource = driver.CreateDataSource(outfile_temp) # create layer - layer = dataSource.CreateLayer(outfile, srs, geom_type=ogr.wkbPolygon) + layer = dataSource.CreateLayer(outfile_temp, srs, geom_type=ogr.wkbPolygon,) # create fields field_id = ogr.FieldDefn("id", ogr.OFTInteger) @@ -257,5 +263,33 @@ def create_geojson_file(geometries, outfile): # add feature to layer layer.CreateFeature(feature) + # make sure to close layer and data source layer = None + dataSource = None + + # load the features from temp file + feature_collection = [] + with open(outfile_temp, "r") as f: + for cnt, line in enumerate(f): + feature_collection.append(json.loads(line)) + + # create final geojson structure + geojson_structure = { + "type": "FeatureCollection", + "name": outfile, + "crs": { + "type": "name", + "properties": {"name": "urn:ogc:def:crs:OGC:1.3:CRS84"}, + }, + "features": feature_collection, + } + # save to geojson + with open(outfile, "w") as json_file: + json.dump(geojson_structure, json_file) + logger.info("created outfile: %s." % outfile) + + # remove temp file + if os.path.exists(outfile_temp): + driver.DeleteDataSource(outfile_temp) + logger.info("created outfile: %s." % outfile) From 5e329354743ae56d5a0aeb6c09518b353a76ed17 Mon Sep 17 00:00:00 2001 From: h9b Date: Tue, 28 Apr 2020 19:57:59 +0200 Subject: [PATCH 3/6] add script to add all geometries for old projects --- .../mapswipe_workers/base/base_project.py | 2 - .../scripts/add_project_geometries_to_api.py | 38 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 mapswipe_workers/scripts/add_project_geometries_to_api.py diff --git a/mapswipe_workers/mapswipe_workers/base/base_project.py b/mapswipe_workers/mapswipe_workers/base/base_project.py index 29ab9ab01..e5bc31e23 100644 --- a/mapswipe_workers/mapswipe_workers/base/base_project.py +++ b/mapswipe_workers/mapswipe_workers/base/base_project.py @@ -372,8 +372,6 @@ def save_to_files(self, project): outfile = ( f"{DATA_PATH}/api/project_geometries/project_geom_{self.projectId}.geojson" ) - print(project) - wkt_geom = project["geometry"] geometries = [ogr.CreateGeometryFromWkt(wkt_geom)] geojson_functions.create_geojson_file(geometries, outfile) diff --git a/mapswipe_workers/scripts/add_project_geometries_to_api.py b/mapswipe_workers/scripts/add_project_geometries_to_api.py new file mode 100644 index 000000000..519350ca2 --- /dev/null +++ b/mapswipe_workers/scripts/add_project_geometries_to_api.py @@ -0,0 +1,38 @@ +from mapswipe_workers.definitions import DATA_PATH +from mapswipe_workers import auth +from mapswipe_workers.utils import geojson_functions +import ogr + + +def add_project_geometries_to_api(): + """Load project geometries from postgres and save as geojson.""" + + # load from postgres + pg_db = auth.postgresDB() + sql_query = """ + SELECT + project_id + ,ST_AsText(geom) as geom + FROM projects + """ + data = pg_db.retr_query(sql_query) + print(len(data)) + + # save as geojson one by one + for i, project in enumerate(data): + project_id = project[0] + wkt_geom = project[1] + + outfile = ( + f"{DATA_PATH}/api/project_geometries/project_geom_{project_id}.geojson" + ) + try: + geometries = [ogr.CreateGeometryFromWkt(wkt_geom)] + geojson_functions.create_geojson_file(geometries, outfile) + except Exception as e: + print(f"got an error for {project_id}") + # just ignore if this fails + pass + + +add_project_geometries_to_api() From 63b69cbefaff6006de77c17f23d513a18f8483ef Mon Sep 17 00:00:00 2001 From: h9b Date: Tue, 28 Apr 2020 21:16:10 +0200 Subject: [PATCH 4/6] add rough idea to upload project to tm --- .../utils/geojson_functions.py | 1 + .../scripts/mapswipe_to_hot_tm_project.py | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 mapswipe_workers/scripts/mapswipe_to_hot_tm_project.py diff --git a/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py b/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py index c8370840b..67fbf9e3a 100644 --- a/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py +++ b/mapswipe_workers/mapswipe_workers/utils/geojson_functions.py @@ -122,6 +122,7 @@ def create_group_geom(group_data, shape="bounding_box"): ring.AddPoint(lon_right, lat_bottom) ring.AddPoint(lon_left, lat_bottom) ring.AddPoint(lon_left, lat_top) + # TODO: Make sure to return 2D geom, currently 3D with z = 0.0 group_geom = ogr.Geometry(ogr.wkbPolygon) group_geom.AddGeometry(ring) diff --git a/mapswipe_workers/scripts/mapswipe_to_hot_tm_project.py b/mapswipe_workers/scripts/mapswipe_to_hot_tm_project.py new file mode 100644 index 000000000..52889582f --- /dev/null +++ b/mapswipe_workers/scripts/mapswipe_to_hot_tm_project.py @@ -0,0 +1,29 @@ +import json +from mapswipe_workers.definitions import DATA_PATH + + +def mapswipe_to_hot_tm_project(project_id, project_name): + + # get project aoi + aoi_file = f"{DATA_PATH}/api/project_geometries/project_geom_{project_id}.geojson" + with open(aoi_file) as f: + aoi_geojson = json.load(f) + + # get tasks + tasks_file = f"{DATA_PATH}/api/hot_tm/hot_tm_{project_id}.geojson" + with open(tasks_file) as f: + tasks_geojson = json.load(f) + + data = { + "arbitraryTasks": True, + "areaOfInterest": aoi_geojson, + "projectName": project_name, + "tasks": tasks_geojson, + } + + print(data) + + +mapswipe_project_id = "build_area_default_with_bing" +tm_project_name = "benni_test" +mapswipe_to_hot_tm_project(mapswpe_project_id, tm_project_name) From 4fd3937b095d72b7342849f3a4fb37aff7a4d0b7 Mon Sep 17 00:00:00 2001 From: h9b Date: Tue, 28 Apr 2020 21:16:50 +0200 Subject: [PATCH 5/6] fix typo --- mapswipe_workers/scripts/mapswipe_to_hot_tm_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/scripts/mapswipe_to_hot_tm_project.py b/mapswipe_workers/scripts/mapswipe_to_hot_tm_project.py index 52889582f..2ac0dac16 100644 --- a/mapswipe_workers/scripts/mapswipe_to_hot_tm_project.py +++ b/mapswipe_workers/scripts/mapswipe_to_hot_tm_project.py @@ -26,4 +26,4 @@ def mapswipe_to_hot_tm_project(project_id, project_name): mapswipe_project_id = "build_area_default_with_bing" tm_project_name = "benni_test" -mapswipe_to_hot_tm_project(mapswpe_project_id, tm_project_name) +mapswipe_to_hot_tm_project(mapswipe_project_id, tm_project_name) From bb1fa99c4c92256a02ff889135e960dbef19e246 Mon Sep 17 00:00:00 2001 From: h9b Date: Thu, 7 May 2020 20:26:05 +0200 Subject: [PATCH 6/6] fix after review --- mapswipe_workers/mapswipe_workers/mapswipe_workers.py | 2 +- mapswipe_workers/scripts/add_project_geometries_to_api.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py index 2e81e0f26..bb96ee81e 100644 --- a/mapswipe_workers/mapswipe_workers/mapswipe_workers.py +++ b/mapswipe_workers/mapswipe_workers/mapswipe_workers.py @@ -83,7 +83,7 @@ def run_create_projects(): project.create_groups() project.calc_required_results() # Save project and its groups and tasks to Firebase and Postgres. - project.save_project + project.save_project() send_slack_message("success", project_name, project.projectId) logger.info("Success: Project Creation ({0})".format(project_name)) except CustomError: diff --git a/mapswipe_workers/scripts/add_project_geometries_to_api.py b/mapswipe_workers/scripts/add_project_geometries_to_api.py index 519350ca2..74f453bb5 100644 --- a/mapswipe_workers/scripts/add_project_geometries_to_api.py +++ b/mapswipe_workers/scripts/add_project_geometries_to_api.py @@ -19,7 +19,7 @@ def add_project_geometries_to_api(): print(len(data)) # save as geojson one by one - for i, project in enumerate(data): + for project in data: project_id = project[0] wkt_geom = project[1] @@ -29,7 +29,7 @@ def add_project_geometries_to_api(): try: geometries = [ogr.CreateGeometryFromWkt(wkt_geom)] geojson_functions.create_geojson_file(geometries, outfile) - except Exception as e: + except Exception: print(f"got an error for {project_id}") # just ignore if this fails pass