Merge pull request #2353 from WikiWatershed/tt/bigcz-cuahsi-detail-va…

…lues-backend

BiG-CZ: Fetch CUAHSI Values using WaterML and Ulmo

Connects #2243
Connects #2238

deployment/ansible/roles.yml

@@ -1,7 +1,7 @@
 - src: azavea.ntp
   version: 0.1.1
 - src: azavea.pip
-  version: 0.1.1
+  version: 1.0.0
 - src: azavea.nodejs
   version: 0.3.0
 - src: azavea.git

src/mmw/apps/bigcz/clients/__init__.py

@@ -22,6 +22,8 @@
         'model': cuahsi.model,
         'serializer': cuahsi.serializer,
         'search': cuahsi.search,
+        'details': cuahsi.details,
+        'values': cuahsi.values,
         'is_pageable': False,
     },
 }

src/mmw/apps/bigcz/clients/cuahsi/__init__.py

@@ -8,6 +8,7 @@
 
 # Import catalog name and search function, so it can be exported from here
 from apps.bigcz.clients.cuahsi.search import CATALOG_NAME, search  # NOQA
+from apps.bigcz.clients.cuahsi.details import details, values  # NOQA
 
 model = CuahsiResource
 serializer = CuahsiResourceSerializer

src/mmw/apps/bigcz/clients/cuahsi/details.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+
+from datetime import date, timedelta
+
+from rest_framework.exceptions import ValidationError
+
+from ulmo.cuahsi import wof
+
+DATE_FORMAT = '%m/%d/%Y'
+
+
+def details(wsdl, site):
+    if not wsdl:
+        raise ValidationError({
+            'error': 'Required argument: wsdl'})
+
+    if not site:
+        raise ValidationError({
+            'error': 'Required argument: site'})
+
+    if not wsdl.upper().endswith('?WSDL'):
+        wsdl += '?WSDL'
+
+    return wof.get_site_info(wsdl, site)
+
+
+def values(wsdl, site, variable, from_date=None, to_date=None):
+    if not wsdl:
+        raise ValidationError({
+            'error': 'Required argument: wsdl'})
+
+    if not site:
+        raise ValidationError({
+            'error': 'Required argument: site'})
+
+    if not variable:
+        raise ValidationError({
+            'error': 'Required argument: variable'})
+
+    if not to_date:
+        # Set to default value of today
+        to_date = date.today().strftime(DATE_FORMAT)
+
+    if not from_date:
+        # Set to default value of one week ago
+        from_date = (date.today() -
+                     timedelta(days=7)).strftime(DATE_FORMAT)
+
+    if not wsdl.upper().endswith('?WSDL'):
+        wsdl += '?WSDL'
+
+    return wof.get_values(wsdl, site, variable, from_date, to_date)

src/mmw/apps/bigcz/clients/cuahsi/models.py

@@ -9,15 +9,16 @@
 class CuahsiResource(Resource):
     def __init__(self, id, description, author, links, title,
                  created_at, updated_at, geom, details_url, sample_mediums,
-                 concept_keywords, service_org, service_code, service_url,
-                 service_title, service_citation, begin_date, end_date):
+                 variables, service_org, service_code, service_url,
+                 service_title, service_citation,
+                 begin_date, end_date):
         super(CuahsiResource, self).__init__(id, description, author, links,
                                              title, created_at, updated_at,
                                              geom)
 
         self.details_url = details_url
         self.sample_mediums = sample_mediums
-        self.concept_keywords = concept_keywords
+        self.variables = variables
         self.service_org = service_org
         self.service_code = service_code
         self.service_url = service_url

src/mmw/apps/bigcz/clients/cuahsi/search.py

@@ -6,7 +6,7 @@
 from datetime import date
 from urllib2 import URLError
 from socket import timeout
-from operator import attrgetter
+from operator import attrgetter, itemgetter
 
 from suds.client import Client
 from suds.sudsobject import asdict
@@ -126,7 +126,7 @@ def parse_record(record, service):
         geom=geom,
         details_url=details_url,
         sample_mediums=record['sample_mediums'],
-        concept_keywords=record['concept_keywords'],
+        variables=record['variables'],
         service_org=service['organization'],
         service_code=record['serv_code'],
         service_url=service['ServiceDescriptionURL'],
@@ -189,6 +189,13 @@ def group_series_by_location(series):
                                for r in group]),
             'end_date': max([parse_date(r['endDate'])
                              for r in group]),
+            'variables': sorted([{
+                'id': r['VarCode'],
+                'name': r['VarName'],
+                'concept_keyword': r['conceptKeyword'],
+                'site': r['location'],
+                'wsdl': r['ServURL'],
+            } for r in group], key=itemgetter('concept_keyword'))
         })
 
     return records

src/mmw/apps/bigcz/clients/cuahsi/serializers.py

@@ -6,15 +6,24 @@
 from rest_framework.serializers import (CharField,
                                         DateTimeField,
                                         ListField,
+                                        Serializer,
                                         )
 
 from apps.bigcz.serializers import ResourceSerializer
 
 
+class CuahsiVariableSetSerializer(Serializer):
+    id = CharField()
+    name = CharField()
+    concept_keyword = CharField()
+    site = CharField()
+    wsdl = CharField()
+
+
 class CuahsiResourceSerializer(ResourceSerializer):
     details_url = CharField()
     sample_mediums = ListField(child=CharField())
-    concept_keywords = ListField(child=CharField())
+    variables = ListField(child=CuahsiVariableSetSerializer())
     service_org = CharField()
     service_code = CharField()
     service_url = CharField()

src/mmw/apps/bigcz/urls.py

@@ -10,4 +10,6 @@
 urlpatterns = patterns(
     '',
     url(r'^search$', views.search, name='bigcz_search'),
+    url(r'^details$', views.details, name='bigcz_details'),
+    url(r'^values$', views.values, name='bigcz_values'),
 )

src/mmw/apps/bigcz/views.py

@@ -8,7 +8,7 @@
 from django.contrib.gis.geos import GEOSGeometry
 from django.conf import settings
 from rest_framework import decorators
-from rest_framework.exceptions import ValidationError, ParseError
+from rest_framework.exceptions import ValidationError, ParseError, NotFound
 from rest_framework.permissions import AllowAny
 from rest_framework.response import Response
 
@@ -87,7 +87,84 @@ def _do_search(request):
         raise ParseError(ex.message)
 
 
+def _get_details(request):
+    params = request.query_params
+    catalog = params.get('catalog')
+
+    if not catalog:
+        raise ValidationError({
+            'error': 'Required argument: catalog'})
+
+    if catalog not in CATALOGS:
+        raise ValidationError({
+            'error': 'Catalog must be one of: {}'
+                     .format(', '.join(CATALOGS.keys()))})
+
+    details = CATALOGS[catalog]['details']
+
+    if not details:
+        raise NotFound({
+            'error': 'No details endpoint for {}'
+                     .format(catalog)})
+
+    details_kwargs = {
+        'wsdl': params.get('wsdl'),
+        'site': params.get('site'),
+    }
+
+    try:
+        return details(**details_kwargs)
+    except ValueError as ex:
+        raise ParseError(ex.message)
+
+
+def _get_values(request):
+    params = request.query_params
+    catalog = params.get('catalog')
+
+    if not catalog:
+        raise ValidationError({
+            'error': 'Required argument: catalog'})
+
+    if catalog not in CATALOGS:
+        raise ValidationError({
+            'error': 'Catalog must be one of: {}'
+                     .format(', '.join(CATALOGS.keys()))})
+
+    values = CATALOGS[catalog]['values']
+
+    if not values:
+        raise NotFound({
+            'error': 'No values endpoint for {}'
+                     .format(catalog)})
+
+    values_kwargs = {
+        'wsdl': params.get('wsdl'),
+        'site': params.get('site'),
+        'variable': params.get('variable'),
+        'from_date': params.get('from_date'),
+        'to_date': params.get('to_date'),
+    }
+
+    try:
+        return values(**values_kwargs)
+    except ValueError as ex:
+        raise ParseError(ex.message)
+
+
 @decorators.api_view(['POST'])
 @decorators.permission_classes((AllowAny,))
 def search(request):
     return Response(_do_search(request))
+
+
+@decorators.api_view(['GET'])
+@decorators.permission_classes((AllowAny,))
+def details(request):
+    return Response(_get_details(request))
+
+
+@decorators.api_view(['GET'])
+@decorators.permission_classes((AllowAny,))
+def values(request):
+    return Response(_get_values(request))

src/mmw/requirements/base.txt

@@ -19,3 +19,4 @@ retry==0.9.1
 python-dateutil==2.6.0
 suds==0.4
 django_celery_results==1.0.1
+ulmo==0.8.4