Merge pull request #2897 from WikiWatershed/tt/hydroshare-celerification

Celerify HydroShare Export

Connects #2885

src/mmw/apps/export/hydroshare.py

@@ -47,9 +47,9 @@ def set_token_from_code(self, code, redirect_uri, user):
 
         return token
 
-    def renew_access_token(self, user):
+    def renew_access_token(self, user_id):
         # TODO Add checks for when user not found, refresh_token is null
-        token = HydroShareToken.objects.get(user=user)
+        token = HydroShareToken.objects.get(user_id=user_id)
         data = {'refresh_token': token.refresh_token,
                 'grant_type': 'refresh_token'}
 
@@ -63,11 +63,11 @@ def renew_access_token(self, user):
 
         return token
 
-    def get_client(self, user):
+    def get_client(self, user_id):
         # TODO Add check for when user not found
-        token = HydroShareToken.objects.get(user=user)
+        token = HydroShareToken.objects.get(user_id=user_id)
         if token.is_expired:
-            token = self.renew_access_token(user)
+            token = self.renew_access_token(user_id)
 
         auth = HydroShareAuthOAuth2(CLIENT_ID, CLIENT_SECRET,
                                     token=token.get_oauth_dict())

src/mmw/apps/export/migrations/0002_hydroshare_disable_autosync.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+def disable_hydroshare_autosync(apps, schema_editor):
+    HydroShareResource = apps.get_model('export', 'HydroShareResource')
+    HydroShareResource.objects.all().update(autosync=False)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('export', '0001_hydroshareresource'),
+    ]
+
+    operations = [
+        migrations.RunPython(disable_hydroshare_autosync)
+    ]

src/mmw/apps/export/tasks.py

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+
+import fiona
+import io
+import json
+import os
+import requests
+
+from celery import shared_task
+
+from django.utils.timezone import now
+from django.contrib.gis.geos import GEOSGeometry
+
+from apps.core.models import Job
+from apps.modeling.models import Project
+from apps.modeling.tasks import to_gms_file
+
+from hydroshare import HydroShareService
+from models import HydroShareResource
+from serializers import HydroShareResourceSerializer
+
+hss = HydroShareService()
+
+SHAPEFILE_EXTENSIONS = ['cpg', 'dbf', 'prj', 'shp', 'shx']
+DEFAULT_KEYWORDS = {'mmw', 'model-my-watershed'}
+MMW_APP_KEY_FLAG = '{"appkey": "model-my-watershed"}'
+BMP_SPREADSHEET_TOOL_URL = 'https://github.com/WikiWatershed/MMW-BMP-spreadsheet-tool/raw/master/MMW_BMP_Spreadsheet_Tool.xlsx'  # NOQA
+
+
+@shared_task(time_limit=300)
+def update_resource(user_id, project_id, params):
+    hs = hss.get_client(user_id)
+    hsresource = HydroShareResource.objects.get(project_id=project_id)
+
+    existing_files = hs.get_file_list(hsresource.resource)
+    if not existing_files:
+        raise RuntimeError('HydroShare could not find requested resource')
+
+    current_analyze_files = [
+        f['url'][(1 + f['url'].index('/analyze_')):]
+        for f in existing_files
+        if '/analyze_' in f['url']
+    ]
+
+    # Update files
+    files = params.get('files', [])
+    for md in params.get('mapshed_data', []):
+        muuid = md.get('uuid')
+        if muuid:
+            try:
+                job = Job.objects.get(uuid=muuid)
+                mdata = json.loads(job.result)
+                files.append({
+                    'name': md.get('name'),
+                    'contents': to_gms_file(mdata),
+                    'object': True,
+                })
+            except (Job.DoesNotExist, ValueError):
+                # Either the job wasn't found, or its content isn't JSON
+                pass
+
+    # Except the existing analyze files
+    files = [f for f in files if f['name'] not in current_analyze_files]
+
+    hs.add_files(hsresource.resource, files, overwrite=True)
+
+    hsresource.exported_at = now()
+    hsresource.save()
+
+    serializer = HydroShareResourceSerializer(hsresource)
+    return serializer.data
+
+
+@shared_task(time_limit=300)
+def create_resource(user_id, project_id, params):
+    hs = hss.get_client(user_id)
+    project = Project.objects.get(pk=project_id)
+
+    # Convert keywords from array to set of values
+    keywords = params.get('keywords')
+    keywords = set(keywords) if keywords else set()
+
+    # POST new resource creates it in HydroShare
+    resource = hs.createResource(
+        'CompositeResource',
+        params.get('title', project.name),
+        abstract=params.get('abstract', ''),
+        keywords=tuple(DEFAULT_KEYWORDS | keywords),
+        extra_metadata=MMW_APP_KEY_FLAG,
+    )
+
+    # Files sent from the client
+    files = params.get('files', [])
+
+    # AoI GeoJSON
+    aoi_geojson = GEOSGeometry(project.area_of_interest).geojson
+    files.append({
+        'name': 'area-of-interest.geojson',
+        'contents': aoi_geojson,
+    })
+
+    # MapShed Data
+    for md in params.get('mapshed_data', []):
+        muuid = md.get('uuid')
+        if muuid:
+            try:
+                job = Job.objects.get(uuid=muuid)
+                mdata = json.loads(job.result)
+                files.append({
+                    'name': md.get('name'),
+                    'contents': to_gms_file(mdata),
+                    'object': True,
+                })
+            except (Job.DoesNotExist, ValueError):
+                # Either the job wasn't found, or its content isn't JSON
+                pass
+
+    # Add all files
+    hs.add_files(resource, files)
+
+    # AoI Shapefile
+    aoi_json = json.loads(aoi_geojson)
+    crs = {'no_defs': True, 'proj': 'longlat',
+           'ellps': 'WGS84', 'datum': 'WGS84'}
+    schema = {'geometry': aoi_json['type'], 'properties': {}}
+    with fiona.open('/tmp/{}.shp'.format(resource), 'w',
+                    driver='ESRI Shapefile',
+                    crs=crs, schema=schema) as shapefile:
+        shapefile.write({'geometry': aoi_json, 'properties': {}})
+
+    for ext in SHAPEFILE_EXTENSIONS:
+        filename = '/tmp/{}.{}'.format(resource, ext)
+        with open(filename) as shapefile:
+            hs.addResourceFile(resource, shapefile,
+                               'area-of-interest.{}'.format(ext))
+        os.remove(filename)
+
+    # MapShed BMP Spreadsheet Tool
+    if params.get('mapshed_data'):
+        response = requests.get(BMP_SPREADSHEET_TOOL_URL,
+                                allow_redirects=True, stream=True)
+        hs.addResourceFile(resource, io.BytesIO(response.content),
+                           'MMW_BMP_Spreadsheet_Tool.xlsx')
+
+    # Make resource public and shareable
+    endpoint = hs.resource(resource)
+    endpoint.public(True)
+    endpoint.shareable(True)
+
+    # Add geographic coverage
+    endpoint.functions.set_file_type({
+        'file_path': 'area-of-interest.shp',
+        'hs_file_type': 'GeoFeature',
+    })
+
+    # Link HydroShareResrouce to Project and save
+    hsresource = HydroShareResource.objects.create(
+        project=project,
+        resource=resource,
+        title=params.get('title', project.name),
+        autosync=params.get('autosync', False),
+        exported_at=now()
+    )
+    hsresource.save()
+
+    # Make Project public and save
+    project.is_private = False
+    project.save()
+
+    # Return newly created HydroShareResource
+    serializer = HydroShareResourceSerializer(hsresource)
+    return serializer.data

src/mmw/apps/export/urls.py

@@ -5,10 +5,14 @@
 
 from django.conf.urls import patterns, url
 
+from apps.modeling.views import get_job
+from apps.modeling.urls import uuid_regex
+
 from apps.export.views import hydroshare, shapefile
 
 urlpatterns = patterns(
     '',
     url(r'^hydroshare/?$', hydroshare, name='hydroshare'),
     url(r'^shapefile/?$', shapefile, name='shapefile'),
+    url(r'jobs/' + uuid_regex, get_job, name='get_job'),
 )