Datasets periodic refresh and centralised Jobs/Datasets memory store …

…management. Fixes ConservationInternational#375

LDMP/__init__.py

@@ -18,6 +18,7 @@
 import site
 import json
 import subprocess
+import datetime
 from tempfile import NamedTemporaryFile
 
 from qgis.PyQt.QtCore import (QSettings, QTranslator, qVersion, 
@@ -133,3 +134,20 @@ def openFolder(path):
         subprocess.check_call(['xdg-open', path])
     elif sys.platform == 'win32':
         subprocess.check_call(['explorer', path])
+
+# singleton decorator
+def singleton(cls):
+    instances = {}
+    def wrapper(*args, **kwargs):
+        if cls not in instances:
+          instances[cls] = cls(*args, **kwargs)
+        return instances[cls]
+    return wrapper
+
+
+def json_serial(obj):
+    """JSON serializer for objects not serializable by default json code"""
+    if isinstance(obj, (datetime.datetime, datetime.date)):
+        return obj.isoformat()
+    raise TypeError("Type {} not serializable".format(type(obj)))
+

LDMP/api.py

@@ -17,12 +17,14 @@
 from builtins import object
 import sys
 import time
+import json
 from datetime import datetime
 from dateutil import tz
 import requests
 import json
 from urllib.parse import quote_plus
 
+import marshmallow
 from qgis.PyQt.QtCore import QCoreApplication, QSettings, QEventLoop
 from qgis.PyQt.QtWidgets import QMessageBox
 
@@ -404,12 +406,54 @@ def register(email, name, organization, country):
     return call_api('/api/v1/user', method='post', payload=payload)
 
 
-def run_script(script, params={}):
+def run_script(script_metadata, params={}):
     # TODO: check before submission whether this payload and script ID has
     # been sent recently - or even whether there are results already
     # available for it. Notify the user if this is the case to prevent, or
     # at least reduce, repeated identical submissions.
-    return call_api(u'/api/v1/script/{}/run'.format(quote_plus(script)), 'post', params, use_token=True)
+    script_name = script_metadata[0]
+    script_slug = script_metadata[1]
+
+    # run a script needs the following steps
+    # 1st step) run it
+    resp = call_api(u'/api/v1/script/{}/run'.format(quote_plus(script_slug)), 'post', params, use_token=True)
+
+    # 2nd step) save returned processing data as json file as process placeholder
+    if resp:
+        # data = resp['data']
+        # only a job should be available as response
+        # if len(data) != 1:
+        #     QMessageBox.critical(None, "Error", tr_api.tr(u"More than one jobs returned running script. Only newest will be get as run representative"))
+        # data = sorted(data, key=lambda job_dict: round(datetime.strptime(job_dict['start_date'], '%Y-%m-%dT%H:%M:%S.%f').timestamp()), reverse=True)
+        job_dict = resp['data']
+
+        # Convert start/end dates into datatime objects in local time zone
+        # the reason is to allog parsin using JobSchema based on APIResponseSchema
+        start_date = datetime.strptime(job_dict['start_date'], '%Y-%m-%dT%H:%M:%S.%f')
+        start_date = start_date.replace(tzinfo=tz.tzutc())
+        start_date = start_date.astimezone(tz.tzlocal())
+        # job_dict['start_date'] = datetime.strftime(start_date, '%Y/%m/%d (%H:%M)')
+        job_dict['start_date'] = start_date
+        end_date = job_dict.get('end_date', None)
+        if end_date:
+            end_date = datetime.strptime(end_date, '%Y-%m-%dT%H:%M:%S.%f')
+            end_date = end_date.replace(tzinfo=tz.tzutc())
+            end_date = end_date.astimezone(tz.tzlocal())
+            # job_dict['end_date'] = datetime.strftime(end_date, '%Y/%m/%d (%H:%M)')
+            job_dict['end_date'] = end_date
+        job_dict['task_name'] = job_dict['params'].get('task_name', '')
+        job_dict['task_notes'] = job_dict['params'].get('task_notes', '')
+        job_dict['params'] = job_dict['params']
+        job_dict['script'] = {"name": script_name, "slug": script_slug}
+
+        # do import here to avoid circular import
+        from LDMP.jobs import Jobs
+        from LDMP.models.datasets import Datasets
+        jobFileName, job = Jobs().append(job_dict)
+        Datasets().appendFromJob(job)
+        Datasets().updated.emit()
+
+    return resp
 
 
 def update_user(email, name, organization, country):
@@ -441,19 +485,23 @@ def get_execution(id=None, date=None):
     if not resp:
         return None
     else:
+        # do import here to avoid circular import
+        from LDMP.jobs import Job, JobSchema
+
         data = resp['data']
         # Sort responses in descending order using start time by default
-        data = sorted(data, key=lambda job: job['start_date'], reverse=True)
+        data = sorted(data, key=lambda job_dict: round(datetime.strptime(job_dict['start_date'], '%Y-%m-%dT%H:%M:%S.%f').timestamp()), reverse=True)
         # Convert start/end dates into datatime objects in local time zone
-        for job in data:
-            start_date = datetime.strptime(job['start_date'], '%Y-%m-%dT%H:%M:%S.%f')
+        for job_dict in data:
+            start_date = datetime.strptime(job_dict['start_date'], '%Y-%m-%dT%H:%M:%S.%f')
             start_date = start_date.replace(tzinfo=tz.tzutc())
             start_date = start_date.astimezone(tz.tzlocal())
-            job['start_date'] = start_date
-            end_date = datetime.strptime(job['end_date'], '%Y-%m-%dT%H:%M:%S.%f')
+            job_dict['start_date'] = start_date
+            end_date = datetime.strptime(job_dict['end_date'], '%Y-%m-%dT%H:%M:%S.%f')
             end_date = end_date.replace(tzinfo=tz.tzutc())
             end_date = end_date.astimezone(tz.tzlocal())
-            job['end_date'] = end_date
+            job_dict['end_date'] = end_date
+
         return data
 
 

LDMP/calculate.py

@@ -74,7 +74,15 @@ def tr(message):
 def get_script_slug(script_name):
     # Note that dots and underscores can't be used in the slugs, so they are 
     # replaced with dashesk
-    return script_name + '-' + scripts[script_name]['script version'].replace('.', '-')
+    return (script_name, script_name + '-' + scripts[script_name]['script version'].replace('.', '-'))
+
+def get_script_group(script_name):
+    # get the configured name of the group that belongs the script
+    group = None
+    if (script_name in scripts) and ('group' in scripts[script_name]):
+        group = scripts[script_name]['group']
+    return group
+
 
 # Transform CRS of a layer while optionally wrapping geometries
 # across the 180th meridian

LDMP/data/scripts.json

@@ -1,15 +1,18 @@
 {
     "time-series": {
         "script version": "1.0.3",
-        "description": "Calculate time series."
+        "description": "Calculate time series.",
+        "group": ""
     },
     "land-cover": {
         "script version": "1.0.3",
-        "description": "Calculate land cover change indicator."
+        "description": "Calculate land cover change indicator.",
+        "group": "SDG 15.3.1"
     },
     "productivity": {
         "script version": "1.0.3",
         "description": "Calculate productivity state, performance, and/or trajectory indicators.",
+        "group": "SDG 15.3.1",
         "trajectory functions": {
             "NDVI trends": {
                 "params": {"trajectory_method": "ndvi_trend"},
@@ -35,26 +38,32 @@
     },
     "soil-organic-carbon": {
         "script version": "1.0.3",
-        "description": "Calculate soil organic carbon."
+        "description": "Calculate soil organic carbon.",
+        "group": "SDG 15.3.1"
     },
     "sdg-sub-indicators": {
         "script version": "1.0.3",
-        "description": "Calculate all three SDG sub-indicators in one step."
+        "description": "Calculate all three SDG sub-indicators in one step.",
+        "group": "SDG 15.3.1"
     },
     "download-data": {
         "script version": "1.0.3",
-        "description": "Download data from Google Earth Engine assets."
+        "description": "Download data from Google Earth Engine assets.",
+        "group": ""
     },
     "total-carbon": {
         "script version": "1.0.3",
-        "description": "Calculate total carbon in biomass (above and below ground)."
+        "description": "Calculate total carbon in biomass (above and below ground).",
+        "group": ""
     },
     "urban-area": {
         "script version": "1.0.3",
-        "description": "Calculate urban area."
+        "description": "Calculate urban area.",
+        "group": ""
     },
     "restoration-biomass": {
         "script version": "1.0.3",
-        "description": "Calculate potential change in biomass with restoration."
+        "description": "Calculate potential change in biomass with restoration.",
+        "group": ""
     }
 }

LDMP/gui/WidgetDatasetItem.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>345</width>
-    <height>152</height>
+    <height>181</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -86,12 +86,9 @@
          </widget>
         </item>
         <item>
-         <widget class="QGraphicsView" name="graphicsViewStatus">
-          <property name="maximumSize">
-           <size>
-            <width>20</width>
-            <height>20</height>
-           </size>
+         <widget class="QPushButton" name="pushButtonStatus">
+          <property name="text">
+           <string/>
           </property>
          </widget>
         </item>