From fb1bdf500e7af85ac00a9d2a08260cc156f26a54 Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 9 Mar 2016 11:20:28 +0700 Subject: [PATCH] Major feature addition: - Add celery shake task to process_shake remotely - Add celery shake task to check existing event - Fix error uninitiated QGIS APP - Push Shake now also push grid.xml file --- .gitignore | 2 +- headless/celery_app.py | 6 +++ realtime/celery_app.py | 4 ++ realtime/earthquake/push_shake.py | 21 ++++++++- realtime/earthquake/shake_event.py | 6 ++- realtime/flood/flood_event.py | 6 ++- realtime/tasks/earthquake.py | 55 ++++++++++++++++++++++- realtime/tasks/flood.py | 5 ++- realtime/test/test_inasafe_django_rest.py | 1 - realtime/test/test_push_rest.py | 1 + 10 files changed, 96 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 3d41b2a402..7545218180 100644 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,4 @@ setup.cfg venv # This is a temporary working file for building the windows installer that does string replacement on version name scripts/build.nsi - +gdalwmscache diff --git a/headless/celery_app.py b/headless/celery_app.py index 8e6bc392b1..56c874a979 100644 --- a/headless/celery_app.py +++ b/headless/celery_app.py @@ -12,6 +12,12 @@ 'headless', ) + +# initialize qgis_app +from safe.test.utilities import get_qgis_app +QGIS_APP, CANVAS, IFACE, PARENT = get_qgis_app() + + app.autodiscover_tasks(packages) if __name__ == '__main__': diff --git a/realtime/celery_app.py b/realtime/celery_app.py index c300ba3007..0e85eb991c 100644 --- a/realtime/celery_app.py +++ b/realtime/celery_app.py @@ -12,6 +12,10 @@ 'realtime', ) +# initialize qgis_app +from safe.test.utilities import get_qgis_app +QGIS_APP, CANVAS, IFACE, PARENT = get_qgis_app() + app.autodiscover_tasks(packages) if __name__ == '__main__': diff --git a/realtime/earthquake/push_shake.py b/realtime/earthquake/push_shake.py index d4c5e6f311..713be2e801 100644 --- a/realtime/earthquake/push_shake.py +++ b/realtime/earthquake/push_shake.py @@ -104,6 +104,11 @@ def push_shake_event_to_rest(shake_event, fail_silent=True): }, 'location_description': event_dict.get('shake-grid-location') } + earthquake_file = { + 'shake_grid': ( + '%s-grid.xml' % shake_event.event_id, + open(shake_event.grid_file_path())), + } # check does the shake event already exists? response = session.earthquake( earthquake_data['shake_id']).GET() @@ -111,11 +116,23 @@ def push_shake_event_to_rest(shake_event, fail_silent=True): # event exists, we should update using PUT Url response = session.earthquake( earthquake_data['shake_id']).PUT( - data=json.dumps(earthquake_data), headers=headers) + data=json.dumps(earthquake_data), + headers=headers) elif response.status_code == requests.codes.not_found: # event does not exists, create using POST url response = session.earthquake.POST( - data=json.dumps(earthquake_data), headers=headers) + data=json.dumps(earthquake_data), + headers=headers) + + # upload grid.xml + headers = { + 'X-CSRFTOKEN': inasafe_django.csrf_token, + } + if response.status_code == requests.codes.ok: + response = session.earthquake( + earthquake_data['shake_id']).PUT( + files=earthquake_file, + headers=headers) if not (response.status_code == requests.codes.ok or response.status_code == requests.codes.created): diff --git a/realtime/earthquake/shake_event.py b/realtime/earthquake/shake_event.py index 7c4cca9c2d..21d79fa79b 100644 --- a/realtime/earthquake/shake_event.py +++ b/realtime/earthquake/shake_event.py @@ -60,10 +60,12 @@ # noinspection PyPackageRequirements from PyQt4.QtXml import QDomDocument +from safe.test.utilities import get_qgis_app +QGIS_APP, CANVAS, IFACE, PARENT = get_qgis_app() + from safe.impact_functions.impact_function_manager import ImpactFunctionManager from safe.storage.core import read_layer as safe_read_layer from safe.engine.core import calculate_impact as safe_calculate_impact -from safe.test.utilities import get_qgis_app from safe.common.version import get_version from safe.common.utilities import romanise from safe.utilities.clipper import extent_to_geoarray, clip_layer @@ -89,8 +91,8 @@ EmptyShakeDirectoryError, EventIdError) + LOGGER = logging.getLogger(realtime_logger_name()) -QGIS_APP, CANVAS, IFACE, PARENT = get_qgis_app() class ShakeEvent(QObject): diff --git a/realtime/flood/flood_event.py b/realtime/flood/flood_event.py index 1bfdb16041..f9fc6a60e7 100644 --- a/realtime/flood/flood_event.py +++ b/realtime/flood/flood_event.py @@ -28,6 +28,10 @@ from realtime.flood.dummy_source_api import DummySourceAPI from realtime.flood.peta_jakarta_api import PetaJakartaAPI from realtime.utilities import realtime_logger_name + +from safe.test.utilities import get_qgis_app +QGIS_APP, CANVAS, IFACE, PARENT = get_qgis_app() + from safe.common.exceptions import ZeroImpactException, TranslationLoadError from safe.engine.core import calculate_impact as safe_calculate_impact from safe.impact_functions.impact_function_manager import \ @@ -35,7 +39,6 @@ from safe.report.impact_report import ImpactReport from safe.storage.safe_layer import SafeLayer from safe.storage.core import read_layer, read_qgis_layer -from safe.test.utilities import get_qgis_app from safe.utilities.keyword_io import KeywordIO from safe.utilities.styling import set_vector_categorized_style, \ set_vector_graduated_style, setRasterStyle @@ -46,7 +49,6 @@ LOGGER = logging.getLogger(realtime_logger_name()) -QGIS_APP, CANVAS, IFACE, PARENT = get_qgis_app() class FloodEvent(QObject): diff --git a/realtime/tasks/earthquake.py b/realtime/tasks/earthquake.py index 01aee85d55..c41c4a8856 100644 --- a/realtime/tasks/earthquake.py +++ b/realtime/tasks/earthquake.py @@ -1,4 +1,57 @@ # coding=utf-8 +import logging + +import os + +from realtime.celery_app import app +from realtime.celeryconfig import EARTHQUAKE_WORKING_DIRECTORY +from realtime.earthquake.make_map import process_event +from realtime.utilities import realtime_logger_name + __author__ = 'Rizky Maulana Nugraha ' -__date__ = '12/11/15' +__date__ = '2/16/16' + + +# Initialized in realtime.__init__ +LOGGER = logging.getLogger(realtime_logger_name()) + + +@app.task( + name='realtime.tasks.earthquake.process_shake', queue='inasafe-realtime') +def process_shake(event_id=None): + LOGGER.info('-------------------------------------------') + + if 'INASAFE_LOCALE' in os.environ: + locale_option = os.environ['INASAFE_LOCALE'] + else: + locale_option = 'en' + + working_directory = EARTHQUAKE_WORKING_DIRECTORY + + if not check_event_exists(event_id): + LOGGER.info('Shake grid not exists') + return False + + try: + process_event(working_directory, event_id, locale_option) + LOGGER.info('Process event end.') + return True + except Exception as e: + LOGGER.exception(e) + + return False + + +@app.task( + name='realtime.tasks.earthquake.check_event_exists', + queue='inasafe-realtime') +def check_event_exists(event_id=None): + LOGGER.info('-------------------------------------------') + + working_directory = EARTHQUAKE_WORKING_DIRECTORY + grid_path = os.path.join( + working_directory, + event_id, + 'output/grid.xml') + return os.path.exists(grid_path) diff --git a/realtime/tasks/flood.py b/realtime/tasks/flood.py index 484b7ead59..563044c69f 100644 --- a/realtime/tasks/flood.py +++ b/realtime/tasks/flood.py @@ -18,7 +18,7 @@ @app.task( name='realtime.tasks.flood.process_flood', queue='inasafe-realtime') -def process_flood(): +def process_flood(event_folder=None): LOGGER.info('-------------------------------------------') if 'INASAFE_LOCALE' in os.environ: @@ -28,7 +28,8 @@ def process_flood(): working_directory = FLOOD_WORKING_DIRECTORY try: - process_event(working_directory, locale_option) + process_event( + working_directory, locale_option, dummy_folder=event_folder) LOGGER.info('Process event end.') return True except Exception as e: diff --git a/realtime/test/test_inasafe_django_rest.py b/realtime/test/test_inasafe_django_rest.py index a3a387997d..ab7e3a818a 100644 --- a/realtime/test/test_inasafe_django_rest.py +++ b/realtime/test/test_inasafe_django_rest.py @@ -2,7 +2,6 @@ import unittest from hammock import Hammock -from requests.models import Response from requests import codes from realtime.push_rest import INASAFE_REALTIME_REST_URL, InaSAFEDjangoREST diff --git a/realtime/test/test_push_rest.py b/realtime/test/test_push_rest.py index cae5bf0164..faf65b0e64 100644 --- a/realtime/test/test_push_rest.py +++ b/realtime/test/test_push_rest.py @@ -5,6 +5,7 @@ from requests import codes from realtime.earthquake.push_shake import notify_realtime_rest +from realtime.earthquake.shake_event import ShakeEvent from realtime.push_rest import InaSAFEDjangoREST __author__ = 'Rizky Maulana Nugraha '