From f1a124c6751b8e0fef1b41012e93ca2e53f8038c Mon Sep 17 00:00:00 2001 From: jgw4sq <~jgw4sq@virginia.edu> Date: Wed, 9 Mar 2016 16:28:14 -0500 Subject: [PATCH 1/8] Created filter so non-normalized documents are no longer displayed with test case --- api/webview/views.py | 7 +++---- tests/test_api_views.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/api/webview/views.py b/api/webview/views.py index bc077f54..4d561932 100644 --- a/api/webview/views.py +++ b/api/webview/views.py @@ -28,9 +28,8 @@ def perform_create(self, serializer): def get_queryset(self): """ Return all documents """ - return Document.objects.all() - - + queryset= Document.objects.all().exclude(normalized=None) + return queryset class DocumentsFromSource(generics.ListAPIView): """ List all documents from a particular source @@ -44,7 +43,7 @@ def perform_create(self, serializer): def get_queryset(self): """ Return queryset based on source """ - return Document.objects.filter(source=self.kwargs['source']) + return Document.objects.filter(source=self.kwargs['source']).exclude(normalized=None) @api_view(['GET']) diff --git a/tests/test_api_views.py b/tests/test_api_views.py index b903140c..e496f450 100644 --- a/tests/test_api_views.py +++ b/tests/test_api_views.py @@ -6,6 +6,7 @@ from rest_framework.test import APIRequestFactory from api.webview.views import DocumentList, status, institutions +from api.webview.models import Document django.setup() @@ -60,4 +61,24 @@ def test_institutions(self): ) response = view(request) self.assertEqual(response.status_code, 200) + def test_exclude_non_normalized_documents(self): + view = DocumentList.as_view() + create_document(source="bad",normalized=None) + create_document(source="good",normalized="This is Normalized") + request = self.factory.get( + '/documents/' + ) + response = view(request) + self.assertNotContains(response, "bad", + status_code=200) + + +def create_document(source,normalized): + return Document.objects.create(source= source,normalized= normalized) + + + + + + From ff288b29ac11dbadcf51aab49012a2080c0cdc2c Mon Sep 17 00:00:00 2001 From: jgw4sq <~jgw4sq@virginia.edu> Date: Wed, 9 Mar 2016 16:43:09 -0500 Subject: [PATCH 2/8] Fixed python formatting errors in previous push of filter for non normalized data --- api/webview/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/webview/views.py b/api/webview/views.py index 4d561932..e8197b1d 100644 --- a/api/webview/views.py +++ b/api/webview/views.py @@ -28,8 +28,10 @@ def perform_create(self, serializer): def get_queryset(self): """ Return all documents """ - queryset= Document.objects.all().exclude(normalized=None) + queryset = Document.objects.all().exclude(normalized=None) return queryset + + class DocumentsFromSource(generics.ListAPIView): """ List all documents from a particular source From 7570a2f5b6747b08aad7c83d7edba5e9c6a161c9 Mon Sep 17 00:00:00 2001 From: jgw4sq <~jgw4sq@virginia.edu> Date: Wed, 9 Mar 2016 16:48:27 -0500 Subject: [PATCH 3/8] fixed more python formatting errors for non normalized filter --- api/webview/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/webview/views.py b/api/webview/views.py index e8197b1d..37a70688 100644 --- a/api/webview/views.py +++ b/api/webview/views.py @@ -31,7 +31,7 @@ def get_queryset(self): queryset = Document.objects.all().exclude(normalized=None) return queryset - + class DocumentsFromSource(generics.ListAPIView): """ List all documents from a particular source From 93bd567cefcfe2faeeb58a0165756bb6a112a94d Mon Sep 17 00:00:00 2001 From: jgw4sq <~jgw4sq@virginia.edu> Date: Thu, 10 Mar 2016 13:22:57 -0500 Subject: [PATCH 4/8] Created new view to be able to display specific documents based on start and end dates in the url. Wrote test case to confirm that this is working --- api/webview/urls.py | 1 + api/webview/views.py | 17 ++++++++++++++++- tests/test_api_views.py | 26 ++++++++++++++++++++++---- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/api/webview/urls.py b/api/webview/urls.py index 38f8c658..297045b5 100644 --- a/api/webview/urls.py +++ b/api/webview/urls.py @@ -7,6 +7,7 @@ url(r'^documents/status', views.status, name='status'), url(r'^documents/(?P\w+)/$', views.DocumentsFromSource.as_view(), name='source'), url(r'^documents/(?P[a-z]+)/(?P(.*))/$', views.document_detail, name='document_detail'), + url(r'^documents/(?P\d{4}-\d{2}-\d{2})&(?P\d{4}-\d{2}-\d{2})/$', views.DocumentsByProviderUpdatedDateTime.as_view(), name='providerupdate'), url(r'^institutions', views.institutions, name='institutions'), url(r'^robots\.txt$', include('robots.urls')), ] diff --git a/api/webview/views.py b/api/webview/views.py index 37a70688..d0733488 100644 --- a/api/webview/views.py +++ b/api/webview/views.py @@ -5,7 +5,7 @@ from rest_framework.response import Response from rest_framework.decorators import api_view from django.views.decorators.clickjacking import xframe_options_exempt - +from dateutil.parser import parse from elasticsearch import Elasticsearch from scrapi import settings @@ -39,6 +39,7 @@ class DocumentsFromSource(generics.ListAPIView): serializer_class = DocumentSerializer permission_classes = (permissions.IsAuthenticatedOrReadOnly,) + def perform_create(self, serializer): serializer.save(source=self.request.user) @@ -47,6 +48,20 @@ def get_queryset(self): """ return Document.objects.filter(source=self.kwargs['source']).exclude(normalized=None) +class DocumentsByProviderUpdatedDateTime(generics.ListAPIView): + """ + List all documents updated within specified time frame + """ + serializer_class = DocumentSerializer + permission_classes = (permissions.IsAuthenticatedOrReadOnly,) + def perform_create(self, serializer): + serializer.save(source=self.request.user) + def get_queryset(self): + """ Return queryset based on source + """ + queryset = Document.objects.all() + queryset = queryset.filter(providerUpdatedDateTime__gte=parse(self.kwargs['from'])).filter(providerUpdatedDateTime__lte=parse(self.kwargs['until'])) + return queryset @api_view(['GET']) @xframe_options_exempt diff --git a/tests/test_api_views.py b/tests/test_api_views.py index e496f450..82d161dc 100644 --- a/tests/test_api_views.py +++ b/tests/test_api_views.py @@ -2,10 +2,10 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "api.api.settings") import django -from django.test import TestCase -from rest_framework.test import APIRequestFactory - -from api.webview.views import DocumentList, status, institutions +from django.test import TestCase +from rest_framework.test import APIRequestFactory, APITestCase, APIClient, force_authenticate +from dateutil.parser import parse +from api.webview.views import DocumentList, status, institutions, DocumentsByProviderUpdatedDateTime from api.webview.models import Document django.setup() @@ -71,8 +71,26 @@ def test_exclude_non_normalized_documents(self): response = view(request) self.assertNotContains(response, "bad", status_code=200) + self.assertContains(response, "good", status_code=200) + print(response) + + +class APIViewTests2(APITestCase): + + def test_query_search_by_providerupdatedtime(self): + view = DocumentsByProviderUpdatedDateTime.as_view() + create_new_document(source = "tooearly",providerUpdatedDateTime= parse("2012-01-01")) + create_new_document(source = "rightontime", providerUpdatedDateTime= parse("2013-01-05")) + create_new_document(source= "toolate", providerUpdatedDateTime = parse("2015-01-01")) + response = self.client.get('/documents/2013-01-01&2014-12-30/', kwargs= {'from':"2013-01-01",'until':'2014-12-30'}) + force_authenticate(response) + self.assertNotContains(response, "tooearly", status_code=200) + self.assertContains(response, "rightontime", status_code=200) + self.assertNotContains(response, "toolate", status_code =200) +def create_new_document(source,providerUpdatedDateTime): + return Document.objects.create(source = source, providerUpdatedDateTime = providerUpdatedDateTime) def create_document(source,normalized): return Document.objects.create(source= source,normalized= normalized) From d4a3877773bc1c00256efd301be36e422a061a4f Mon Sep 17 00:00:00 2001 From: jgw4sq <~jgw4sq@virginia.edu> Date: Thu, 10 Mar 2016 13:48:23 -0500 Subject: [PATCH 5/8] Made minor changes to format or the url for query by providerupdate time --- api/webview/urls.py | 2 +- tests/test_api_views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/webview/urls.py b/api/webview/urls.py index 297045b5..8715adb9 100644 --- a/api/webview/urls.py +++ b/api/webview/urls.py @@ -7,7 +7,7 @@ url(r'^documents/status', views.status, name='status'), url(r'^documents/(?P\w+)/$', views.DocumentsFromSource.as_view(), name='source'), url(r'^documents/(?P[a-z]+)/(?P(.*))/$', views.document_detail, name='document_detail'), - url(r'^documents/(?P\d{4}-\d{2}-\d{2})&(?P\d{4}-\d{2}-\d{2})/$', views.DocumentsByProviderUpdatedDateTime.as_view(), name='providerupdate'), + url(r'^documents/from=(?P\d{4}-\d{2}-\d{2})&until=(?P\d{4}-\d{2}-\d{2})/$', views.DocumentsByProviderUpdatedDateTime.as_view(), name='providerupdate'), url(r'^institutions', views.institutions, name='institutions'), url(r'^robots\.txt$', include('robots.urls')), ] diff --git a/tests/test_api_views.py b/tests/test_api_views.py index 82d161dc..0f26dd74 100644 --- a/tests/test_api_views.py +++ b/tests/test_api_views.py @@ -82,7 +82,7 @@ def test_query_search_by_providerupdatedtime(self): create_new_document(source = "tooearly",providerUpdatedDateTime= parse("2012-01-01")) create_new_document(source = "rightontime", providerUpdatedDateTime= parse("2013-01-05")) create_new_document(source= "toolate", providerUpdatedDateTime = parse("2015-01-01")) - response = self.client.get('/documents/2013-01-01&2014-12-30/', kwargs= {'from':"2013-01-01",'until':'2014-12-30'}) + response = self.client.get('/documents/from=2013-01-01&until=2014-12-30/', kwargs= {'from':"2013-01-01",'until':'2014-12-30'}) force_authenticate(response) self.assertNotContains(response, "tooearly", status_code=200) self.assertContains(response, "rightontime", status_code=200) From 1393a44b3c4b31c9e2a9ce85eb41dd264771bc0d Mon Sep 17 00:00:00 2001 From: jgw4sq <~jgw4sq@virginia.edu> Date: Thu, 10 Mar 2016 17:06:41 -0500 Subject: [PATCH 6/8] Made python formatting changes to previous commit --- api/webview/views.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/webview/views.py b/api/webview/views.py index d0733488..a2a79848 100644 --- a/api/webview/views.py +++ b/api/webview/views.py @@ -38,8 +38,7 @@ class DocumentsFromSource(generics.ListAPIView): """ serializer_class = DocumentSerializer permission_classes = (permissions.IsAuthenticatedOrReadOnly,) - - + def perform_create(self, serializer): serializer.save(source=self.request.user) @@ -48,14 +47,17 @@ def get_queryset(self): """ return Document.objects.filter(source=self.kwargs['source']).exclude(normalized=None) + class DocumentsByProviderUpdatedDateTime(generics.ListAPIView): """ List all documents updated within specified time frame """ serializer_class = DocumentSerializer permission_classes = (permissions.IsAuthenticatedOrReadOnly,) + def perform_create(self, serializer): serializer.save(source=self.request.user) + def get_queryset(self): """ Return queryset based on source """ @@ -63,6 +65,7 @@ def get_queryset(self): queryset = queryset.filter(providerUpdatedDateTime__gte=parse(self.kwargs['from'])).filter(providerUpdatedDateTime__lte=parse(self.kwargs['until'])) return queryset + @api_view(['GET']) @xframe_options_exempt def document_detail(request, source, docID): From 85f8e8dc5a8a37b7007d2523fdc0d0741c3213e8 Mon Sep 17 00:00:00 2001 From: jgw4sq <~jgw4sq@virginia.edu> Date: Fri, 11 Mar 2016 08:53:09 -0500 Subject: [PATCH 7/8] Fixed python formatting errors from previous commit --- api/webview/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/webview/views.py b/api/webview/views.py index a2a79848..93a8fec7 100644 --- a/api/webview/views.py +++ b/api/webview/views.py @@ -54,12 +54,12 @@ class DocumentsByProviderUpdatedDateTime(generics.ListAPIView): """ serializer_class = DocumentSerializer permission_classes = (permissions.IsAuthenticatedOrReadOnly,) - + def perform_create(self, serializer): serializer.save(source=self.request.user) - + def get_queryset(self): - """ Return queryset based on source + """ Return queryset based on provider update time """ queryset = Document.objects.all() queryset = queryset.filter(providerUpdatedDateTime__gte=parse(self.kwargs['from'])).filter(providerUpdatedDateTime__lte=parse(self.kwargs['until'])) From ba1deec9def7cac26a238caa297c5df17fdbd80e Mon Sep 17 00:00:00 2001 From: jgw4sq <~jgw4sq@virginia.edu> Date: Fri, 11 Mar 2016 09:24:37 -0500 Subject: [PATCH 8/8] Fixed more python formatting errors from previous commit --- api/webview/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/webview/views.py b/api/webview/views.py index 93a8fec7..ca1aadc9 100644 --- a/api/webview/views.py +++ b/api/webview/views.py @@ -38,7 +38,7 @@ class DocumentsFromSource(generics.ListAPIView): """ serializer_class = DocumentSerializer permission_classes = (permissions.IsAuthenticatedOrReadOnly,) - + def perform_create(self, serializer): serializer.save(source=self.request.user)