From c4238384ad7f39e01fc0af5c6937c6e16c6cc879 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 14 Jul 2017 16:01:33 -0400 Subject: [PATCH 1/4] Move clients from files to packages This is done in anticipation of one or more of these clients needing their own models and serializers. :hammer: --- src/mmw/apps/bigcz/clients/__init__.py | 23 ++++++++++++++----- .../apps/bigcz/clients/cinergi/__init__.py | 13 +++++++++++ .../clients/{cinergi.py => cinergi/search.py} | 4 ++-- src/mmw/apps/bigcz/clients/cuahsi/__init__.py | 13 +++++++++++ .../clients/{cuahsi.py => cuahsi/search.py} | 4 ++-- .../apps/bigcz/clients/hydroshare/__init__.py | 13 +++++++++++ .../{hydroshare.py => hydroshare/search.py} | 4 ++-- src/mmw/apps/bigcz/views.py | 23 +++++++++---------- 8 files changed, 73 insertions(+), 24 deletions(-) create mode 100644 src/mmw/apps/bigcz/clients/cinergi/__init__.py rename src/mmw/apps/bigcz/clients/{cinergi.py => cinergi/search.py} (96%) create mode 100644 src/mmw/apps/bigcz/clients/cuahsi/__init__.py rename src/mmw/apps/bigcz/clients/{cuahsi.py => cuahsi/search.py} (97%) create mode 100644 src/mmw/apps/bigcz/clients/hydroshare/__init__.py rename src/mmw/apps/bigcz/clients/{hydroshare.py => hydroshare/search.py} (95%) diff --git a/src/mmw/apps/bigcz/clients/__init__.py b/src/mmw/apps/bigcz/clients/__init__.py index 25027affd..b830171a8 100644 --- a/src/mmw/apps/bigcz/clients/__init__.py +++ b/src/mmw/apps/bigcz/clients/__init__.py @@ -3,11 +3,22 @@ from __future__ import unicode_literals from __future__ import division -from apps.bigcz.clients import hydroshare, cuahsi, cinergi +from apps.bigcz.clients import cinergi, hydroshare, cuahsi - -SEARCH_FUNCTIONS = { - hydroshare.CATALOG_NAME: hydroshare.search, - cuahsi.CATALOG_NAME: cuahsi.search, - cinergi.CATALOG_NAME: cinergi.search, +CATALOGS = { + cinergi.CATALOG_NAME: { + 'model': cinergi.model, + 'serializer': cinergi.serializer, + 'search': cinergi.search, + }, + hydroshare.CATALOG_NAME: { + 'model': hydroshare.model, + 'serializer': hydroshare.serializer, + 'search': hydroshare.search, + }, + cuahsi.CATALOG_NAME: { + 'model': cuahsi.model, + 'serializer': cuahsi.serializer, + 'search': cuahsi.search, + }, } diff --git a/src/mmw/apps/bigcz/clients/cinergi/__init__.py b/src/mmw/apps/bigcz/clients/cinergi/__init__.py new file mode 100644 index 000000000..670b2c00c --- /dev/null +++ b/src/mmw/apps/bigcz/clients/cinergi/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division + +from apps.bigcz.models import Resource +from apps.bigcz.serializers import ResourceSerializer + +# Import catalog name and search function, so it can be exported from here +from apps.bigcz.clients.cinergi.search import CATALOG_NAME, search # NOQA + +model = Resource +serializer = ResourceSerializer diff --git a/src/mmw/apps/bigcz/clients/cinergi.py b/src/mmw/apps/bigcz/clients/cinergi/search.py similarity index 96% rename from src/mmw/apps/bigcz/clients/cinergi.py rename to src/mmw/apps/bigcz/clients/cinergi/search.py index e7b64e0ae..e2633ca9f 100644 --- a/src/mmw/apps/bigcz/clients/cinergi.py +++ b/src/mmw/apps/bigcz/clients/cinergi/search.py @@ -14,7 +14,7 @@ CATALOG_NAME = 'cinergi' -GEOPORTAL_URL = 'http://132.249.238.169:8080/geoportal/opensearch' +CATALOG_URL = 'http://132.249.238.169:8080/geoportal/opensearch' def parse_date(value): @@ -132,7 +132,7 @@ def search(**kwargs): }) try: - response = requests.get(GEOPORTAL_URL, + response = requests.get(CATALOG_URL, timeout=settings.BIGCZ_CLIENT_TIMEOUT, params=params) except requests.Timeout: diff --git a/src/mmw/apps/bigcz/clients/cuahsi/__init__.py b/src/mmw/apps/bigcz/clients/cuahsi/__init__.py new file mode 100644 index 000000000..5eb956acb --- /dev/null +++ b/src/mmw/apps/bigcz/clients/cuahsi/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division + +from apps.bigcz.models import Resource +from apps.bigcz.serializers import ResourceSerializer + +# Import catalog name and search function, so it can be exported from here +from apps.bigcz.clients.cuahsi.search import CATALOG_NAME, search # NOQA + +model = Resource +serializer = ResourceSerializer diff --git a/src/mmw/apps/bigcz/clients/cuahsi.py b/src/mmw/apps/bigcz/clients/cuahsi/search.py similarity index 97% rename from src/mmw/apps/bigcz/clients/cuahsi.py rename to src/mmw/apps/bigcz/clients/cuahsi/search.py index 47b6eacfe..5072795d2 100644 --- a/src/mmw/apps/bigcz/clients/cuahsi.py +++ b/src/mmw/apps/bigcz/clients/cuahsi/search.py @@ -20,7 +20,7 @@ SQKM_PER_SQM = 0.000001 MAX_AREA_SQKM = 1500 CATALOG_NAME = 'cuahsi' -SOAP_URL = 'http://hiscentral.cuahsi.org/webservices/hiscentral.asmx?WSDL' +CATALOG_URL = 'http://hiscentral.cuahsi.org/webservices/hiscentral.asmx?WSDL' DATE_MIN = date(1900, 1, 1) @@ -28,7 +28,7 @@ DATE_FORMAT = '%m/%d/%Y' -client = Client(SOAP_URL, timeout=settings.BIGCZ_CLIENT_TIMEOUT) +client = Client(CATALOG_URL, timeout=settings.BIGCZ_CLIENT_TIMEOUT) def parse_geom(record): diff --git a/src/mmw/apps/bigcz/clients/hydroshare/__init__.py b/src/mmw/apps/bigcz/clients/hydroshare/__init__.py new file mode 100644 index 000000000..324083e83 --- /dev/null +++ b/src/mmw/apps/bigcz/clients/hydroshare/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division + +from apps.bigcz.models import Resource +from apps.bigcz.serializers import ResourceSerializer + +# Import catalog name and search function, so it can be exported from here +from apps.bigcz.clients.hydroshare.search import CATALOG_NAME, search # NOQA + +model = Resource +serializer = ResourceSerializer diff --git a/src/mmw/apps/bigcz/clients/hydroshare.py b/src/mmw/apps/bigcz/clients/hydroshare/search.py similarity index 95% rename from src/mmw/apps/bigcz/clients/hydroshare.py rename to src/mmw/apps/bigcz/clients/hydroshare/search.py index 64058643a..75af0ab14 100644 --- a/src/mmw/apps/bigcz/clients/hydroshare.py +++ b/src/mmw/apps/bigcz/clients/hydroshare/search.py @@ -14,7 +14,7 @@ CATALOG_NAME = 'hydroshare' -HYDROSHARE_URL = 'https://www.hydroshare.org/hsapi/resource/' +CATALOG_URL = 'https://www.hydroshare.org/hsapi/resource/' def parse_date(value): @@ -75,7 +75,7 @@ def search(**kwargs): params.update(prepare_bbox(bbox)) try: - response = requests.get(HYDROSHARE_URL, + response = requests.get(CATALOG_URL, timeout=settings.BIGCZ_CLIENT_TIMEOUT, params=params) except requests.Timeout: diff --git a/src/mmw/apps/bigcz/views.py b/src/mmw/apps/bigcz/views.py index 838e46b09..1408070d8 100644 --- a/src/mmw/apps/bigcz/views.py +++ b/src/mmw/apps/bigcz/views.py @@ -8,7 +8,7 @@ from rest_framework.permissions import AllowAny from rest_framework.response import Response -from apps.bigcz.clients import SEARCH_FUNCTIONS +from apps.bigcz.clients import CATALOGS from apps.bigcz.serializers import ResourceListSerializer from apps.bigcz.utils import parse_date @@ -20,6 +20,11 @@ def _do_search(request): raise ValidationError({ 'error': 'Required argument: catalog'}) + if catalog not in CATALOGS: + raise ValidationError({ + 'error': 'Catalog must be one of: {}' + .format(', '.join(CATALOGS.keys()))}) + search_kwargs = { 'query': request.query_params.get('query'), 'to_date': parse_date(request.query_params.get('to_date')), @@ -27,18 +32,12 @@ def _do_search(request): 'bbox': request.query_params.get('bbox'), } - search = SEARCH_FUNCTIONS.get(catalog) - - if search: - try: - return search(**search_kwargs) - except ValueError as ex: - raise ParseError(ex.message) + search = CATALOGS[catalog]['search'] - raise ValidationError({ - 'error': 'Catalog must be one of: {}' - .format(', '.join(SEARCH_FUNCTIONS.keys())) - }) + try: + return search(**search_kwargs) + except ValueError as ex: + raise ParseError(ex.message) @decorators.api_view(['GET']) From 3582796979be3d8b2df5216ad5926d75ca2055e1 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 14 Jul 2017 16:41:49 -0400 Subject: [PATCH 2/4] Dynamic serialization of resource lists In anticipation of providing a serializer at runtime for the various clients, we change the ResourceListSerializer.results field to be dynamic. If given a `serializer` in the context object, we will use that to serialize the results, falling back to the generic serializer. I'm not sure if this is the most performant way of doing it, but we can focus on optimizations at a later stage. See: * http://www.django-rest-framework.org/api-guide/fields/#serializermethodfield * http://www.django-rest-framework.org/api-guide/serializers/#including-extra-context :hammer: --- src/mmw/apps/bigcz/serializers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/mmw/apps/bigcz/serializers.py b/src/mmw/apps/bigcz/serializers.py index e47d82263..94d6ed49d 100644 --- a/src/mmw/apps/bigcz/serializers.py +++ b/src/mmw/apps/bigcz/serializers.py @@ -4,7 +4,7 @@ from __future__ import division from rest_framework.serializers import \ - Serializer, CharField, DateTimeField, IntegerField + Serializer, CharField, DateTimeField, IntegerField, SerializerMethodField from rest_framework_gis.serializers import GeometryField @@ -27,5 +27,9 @@ class ResourceSerializer(Serializer): class ResourceListSerializer(Serializer): catalog = CharField() api_url = CharField() - results = ResourceSerializer(many=True) + results = SerializerMethodField() count = IntegerField() + + def get_results(self, obj): + serializer = self.context.get('serializer', ResourceSerializer) + return [serializer(r).data for r in obj.results] From 86e734d83786ecca459645fdf23fb81762834a40 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 14 Jul 2017 17:01:26 -0400 Subject: [PATCH 3/4] Pass dynamic serializer For each client, use the appropriate serializer :hammer: --- src/mmw/apps/bigcz/views.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mmw/apps/bigcz/views.py b/src/mmw/apps/bigcz/views.py index 1408070d8..bbcd46f78 100644 --- a/src/mmw/apps/bigcz/views.py +++ b/src/mmw/apps/bigcz/views.py @@ -33,9 +33,12 @@ def _do_search(request): } search = CATALOGS[catalog]['search'] + serializer = CATALOGS[catalog]['serializer'] try: - return search(**search_kwargs) + result = ResourceListSerializer(search(**search_kwargs), + context={'serializer': serializer}) + return result.data except ValueError as ex: raise ParseError(ex.message) @@ -43,5 +46,4 @@ def _do_search(request): @decorators.api_view(['GET']) @decorators.permission_classes((AllowAny,)) def search(request): - result = ResourceListSerializer(_do_search(request)) - return Response(result.data) + return Response(_do_search(request)) From c5ebd7649d211219885f2fff2bd2c49ed576130a Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 14 Jul 2017 17:18:26 -0400 Subject: [PATCH 4/4] Add custom model and serializer for CUAHSI Also add a test field to demo the difference. --- src/mmw/apps/bigcz/clients/cuahsi/__init__.py | 8 ++++---- src/mmw/apps/bigcz/clients/cuahsi/models.py | 16 ++++++++++++++++ src/mmw/apps/bigcz/clients/cuahsi/search.py | 9 ++++++--- src/mmw/apps/bigcz/clients/cuahsi/serializers.py | 12 ++++++++++++ 4 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 src/mmw/apps/bigcz/clients/cuahsi/models.py create mode 100644 src/mmw/apps/bigcz/clients/cuahsi/serializers.py diff --git a/src/mmw/apps/bigcz/clients/cuahsi/__init__.py b/src/mmw/apps/bigcz/clients/cuahsi/__init__.py index 5eb956acb..57f81b9fb 100644 --- a/src/mmw/apps/bigcz/clients/cuahsi/__init__.py +++ b/src/mmw/apps/bigcz/clients/cuahsi/__init__.py @@ -3,11 +3,11 @@ from __future__ import unicode_literals from __future__ import division -from apps.bigcz.models import Resource -from apps.bigcz.serializers import ResourceSerializer +from apps.bigcz.clients.cuahsi.models import CuahsiResource +from apps.bigcz.clients.cuahsi.serializers import CuahsiResourceSerializer # Import catalog name and search function, so it can be exported from here from apps.bigcz.clients.cuahsi.search import CATALOG_NAME, search # NOQA -model = Resource -serializer = ResourceSerializer +model = CuahsiResource +serializer = CuahsiResourceSerializer diff --git a/src/mmw/apps/bigcz/clients/cuahsi/models.py b/src/mmw/apps/bigcz/clients/cuahsi/models.py new file mode 100644 index 000000000..b4514967f --- /dev/null +++ b/src/mmw/apps/bigcz/clients/cuahsi/models.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division + +from apps.bigcz.models import Resource + + +class CuahsiResource(Resource): + def __init__(self, id, description, author, links, title, + created_at, updated_at, geom, test_field): + super(CuahsiResource, self).__init__(id, description, author, links, + title, created_at, updated_at, + geom) + + self.test_field = test_field diff --git a/src/mmw/apps/bigcz/clients/cuahsi/search.py b/src/mmw/apps/bigcz/clients/cuahsi/search.py index 5072795d2..08622b416 100644 --- a/src/mmw/apps/bigcz/clients/cuahsi/search.py +++ b/src/mmw/apps/bigcz/clients/cuahsi/search.py @@ -13,9 +13,11 @@ from django.conf import settings -from apps.bigcz.models import Resource, ResourceLink, ResourceList, BBox +from apps.bigcz.models import ResourceLink, ResourceList, BBox from apps.bigcz.utils import parse_date, RequestTimedOutError +from apps.bigcz.clients.cuahsi.models import CuahsiResource + SQKM_PER_SQM = 0.000001 MAX_AREA_SQKM = 1500 @@ -65,7 +67,7 @@ def parse_record(record, service): if details_url: links.append(ResourceLink('details', details_url)) - return Resource( + return CuahsiResource( id=record['location'], title=record['Sitename'], description=service['aabstract'], @@ -73,7 +75,8 @@ def parse_record(record, service): links=links, created_at=record['beginDate'], updated_at=None, - geom=geom) + geom=geom, + test_field="hello world") def find_service(services, service_code): diff --git a/src/mmw/apps/bigcz/clients/cuahsi/serializers.py b/src/mmw/apps/bigcz/clients/cuahsi/serializers.py new file mode 100644 index 000000000..42de8d447 --- /dev/null +++ b/src/mmw/apps/bigcz/clients/cuahsi/serializers.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division + +from rest_framework.serializers import CharField + +from apps.bigcz.serializers import ResourceSerializer + + +class CuahsiResourceSerializer(ResourceSerializer): + test_field = CharField()