Request multiple cores, check validity of cores and return fields

.gitignore

@@ -71,4 +71,6 @@ venv.bak/
 .mypy_cache/
 
 # nltk
-averaged_perceptron_tagger/
+averaged_perceptron_tagger/
+
+*.DS_Store

sleuth_backend/solr/models.py

@@ -93,3 +93,14 @@ class RedditPost(SolrDocument):
 
     def __init__(self, **kwargs):
         super(RedditPost, self).__init__(self.doc, **kwargs)
+
+models = [SolrDocument, GenericPage, CourseItem, RedditPost]
+
+def get_models_fields():
+    '''
+    Return a list of the doc description of each model type
+    '''
+    fields = []
+    for d in [m.doc for m in models]:
+        fields += [k for k, _ in d.items() if k not in fields]
+    return fields

sleuth_backend/tests/test_models.py

@@ -1,5 +1,5 @@
 from django.test import TestCase
-from sleuth_backend.solr.models import SolrDocument, GenericPage, CourseItem
+from sleuth_backend.solr.models import SolrDocument, GenericPage, CourseItem, get_models_fields
 from unittest.mock import MagicMock
 
 class TestModels(TestCase):
@@ -48,3 +48,10 @@ def test_save_to_solr(self):
         mock.queue_document.return_value = None
         page.save_to_solr(mock)
         mock.queue_document.assert_called_with("genericPage", args)
+
+    def test_get_models_fields(self):
+        result = get_models_fields()
+        self.assertEquals(
+            ['id', 'type', 'name', 'updatedAt', 'description', 'siteName', 'content', 'links', 'subjectId', 'subjectName', 'faculty', 'comments', 'subreddit'],
+            result
+        )

sleuth_backend/tests/test_views.py

@@ -61,6 +61,8 @@ def test_apis_without_params(self, mock_query):
     @patch('sleuth_backend.solr.connection.SolrConnection.core_names')
     @patch('sleuth_backend.solr.connection.SolrConnection.query')
     def test_apis_with_valid_request(self, mock_query, mock_cores):
+        mock_cores.return_value = ['genericPage', 'redditPost', 'courseItem']
+
         # genericPage search
         mock_query.return_value = {
             "type": "genericPage",
@@ -93,6 +95,7 @@ def test_apis_with_valid_request(self, mock_query, mock_cores):
         mock_response['response']['docs'][0]['updatedAt'] = ''
         mock_response['response']['docs'][0]['name'] = ''
         mock_response['response']['docs'][0]['description'] = 'Nice one dude'
+        self.maxDiff = None
         self.assertEqual(
             json.loads(result.content.decode('utf-8')),
             {
@@ -124,6 +127,7 @@ def test_apis_with_valid_request(self, mock_query, mock_cores):
         mock_response = mock_query.return_value
 
         # getdocument
+        mock_cores.return_value = ['genericPage', 'redditPost', 'courseItem']        
         params = {
             'id': 'somequery',
             'type': 'genericPage',
@@ -143,8 +147,11 @@ def test_apis_with_valid_request(self, mock_query, mock_cores):
         result = getdocument(mock_request)
         self.assertEqual(result.status_code, 404)
 
+    @patch('sleuth_backend.solr.connection.SolrConnection.core_names')
     @patch('sleuth_backend.solr.connection.SolrConnection.query')
-    def test_apis_with_error_response(self, mock_query):
+    def test_apis_with_error_response(self, mock_query, mock_cores):
+        mock_cores.return_value = ['test']        
+
         # Solr response error
         mock_query.return_value = {
             "error": {

sleuth_backend/tests/test_views_utils.py

@@ -0,0 +1,40 @@
+import sleuth_backend.views.views_utils as utils
+from django.test import TestCase
+
+class TestViewsUtils(TestCase):
+
+    def test_build_core_request(self):
+        solr_cores = ['genericPage', 'redditPost', 'courseItem']
+
+        # one requested core
+        result = utils.build_core_request('genericPage', solr_cores)
+        self.assertEquals(['genericPage'], result)        
+
+        # multiple requested cores
+        result = utils.build_core_request('genericPage,redditPost', solr_cores)
+        self.assertEquals(['genericPage', 'redditPost'], result)
+
+        # some invalid cores
+        result = utils.build_core_request('wow,redditPost', solr_cores)
+        self.assertEquals(['redditPost'], result)
+
+        # all cores invalid
+        result = utils.build_core_request('geasdficPage,redasdf', solr_cores)
+        self.assertEquals(solr_cores, result)
+
+        # no given cores
+        result = utils.build_core_request('', solr_cores)
+        self.assertEquals(solr_cores, result)
+
+    def test_build_return_fields(self):
+        result = utils.build_return_fields('')
+        self.assertEquals('id,updatedAt,name,description', result)
+
+        result = utils.build_return_fields('asdfasdf')
+        self.assertEquals('id,updatedAt,name,description', result)
+
+        result = utils.build_return_fields('links,sadf')
+        self.assertEquals('id,updatedAt,name,description,links', result)
+
+        result = utils.build_return_fields('asdf,links,subreddit')
+        self.assertEquals('id,updatedAt,name,description,links,subreddit', result)

sleuth_backend/views/views.py

@@ -5,8 +5,8 @@
 import pysolr
 import json
 from django.http import HttpResponse, JsonResponse
+import sleuth_backend.views.views_utils as utils
 from .error import SleuthError, ErrorTypes
-from .views_utils import *
 from sleuth_backend.solr import connection as solr
 from sleuth.settings import HAYSTACK_CONNECTIONS
 
@@ -80,10 +80,10 @@ def search(request):
     if request.method != 'GET':
         return HttpResponse(status=405)
 
-    cores_to_search = build_core_request(request.GET.get('type', ''), SOLR.core_names())
+    cores_to_search = utils.build_core_request(request.GET.get('type', ''), SOLR.core_names())
     query = request.GET.get('q', '')
     state = request.GET.get('state', '')
-    return_fields = build_return_fields(request.GET.get('return', ''))
+    return_fields = utils.build_return_fields(request.GET.get('return', ''))
 
     kwargs = {
         'sort': request.GET.get('sort', ''),
@@ -107,7 +107,7 @@ def search(request):
     }
     for core_to_search in cores_to_search:
         try:
-            new_query, new_kwargs = build_search_query(core_to_search, query, kwargs)
+            new_query, new_kwargs = utils.build_search_query(core_to_search, query, kwargs)
             query_response = SOLR.query(core_to_search, new_query, **new_kwargs)
             if 'error' in query_response:
                 sleuth_error = SleuthError(
@@ -119,13 +119,13 @@ def search(request):
             # Attach type to response and flatten single-item list fields
             query_response['type'] = core_to_search
             for doc in query_response['response']['docs']:
-                flatten_doc(doc, return_fields)
+                utils.flatten_doc(doc, return_fields)
 
             responses['data'].append(query_response)
 
         # Handle errors and exceptions from each query
         except (Exception, pysolr.SolrError) as e:
-            message, status = build_error(e)
+            message, status = utils.build_error(e)
             return HttpResponse(message, status=status)
 
     return JsonResponse(responses)
@@ -156,10 +156,10 @@ def getdocument(request):
     if request.method != 'GET':
         return HttpResponse(status=405)
 
-    cores_to_search = build_core_request(request.GET.get('type', ''), SOLR.core_names())
+    cores_to_search = utils.build_core_request(request.GET.get('type', ''), SOLR.core_names())
     doc_id = request.GET.get('id', '')
     state = request.GET.get('state', '')
-    return_fields = build_return_fields(request.GET.get('return', ''))
+    return_fields = utils.build_return_fields(request.GET.get('return', ''))
 
     kwargs = { 'return_fields': return_fields }
 
@@ -178,7 +178,7 @@ def getdocument(request):
     }
     for core_to_search in cores_to_search:
         try:
-            new_query, new_kwargs = build_getdocument_query(doc_id, kwargs)
+            new_query, new_kwargs = utils.build_getdocument_query(doc_id, kwargs)
             query_response = SOLR.query(core_to_search, new_query, **new_kwargs)
             if 'error' in query_response:
                 sleuth_error = SleuthError(
@@ -190,13 +190,13 @@ def getdocument(request):
                 # Return result if a value is found
                 response['data'] = {
                     'type': core_to_search,
-                    'doc': flatten_doc(query_response['response']['docs'][0], return_fields)
+                    'doc': utils.flatten_doc(query_response['response']['docs'][0], return_fields)
                 }
                 return JsonResponse(response)
 
         # Handle errors and exceptions from each query
         except (Exception, pysolr.SolrError) as e:
-            message, status = build_error(e)
+            message, status = utils.build_error(e)
             return HttpResponse(message, status=status)
 
     return HttpResponse("Document not found", status=404)

sleuth_backend/views/views_utils.py

@@ -5,20 +5,29 @@
 from pysolr import SolrError
 from .error import SleuthError, ErrorTypes
 from sleuth_backend.solr.query import Query
+import sleuth_backend.solr.models as models
 
 def build_core_request(core, solr_cores):
     '''
     Builds a list of cores to search based on given core parameter
+    Also checks requested cores against available cores and discards
+    invalid requested cores
     '''
-    return [c for c in solr_cores] if core is '' else [core]
+    core_params = [s for s in core.split(',')]
+    core_params = [c for c in core_params if c in solr_cores]
+    return solr_cores if len(core_params) is 0 else core_params
 
 def build_return_fields(fields):
     '''
     Builds a string listing the fields to return
+    Also checks requested return fields against available return fields
+    and discards invalid return fields
     '''
     return_fields = 'id,updatedAt,name,description'
-    if fields is not '':
-        return_fields = return_fields + ',' + fields
+    fields_list = [s for s in fields.split(',')]
+    fields_list = [f for f in fields_list if f in models.get_models_fields()]
+    if len(fields_list) > 0:
+        return_fields = return_fields + ',' + ",".join(fields_list)
     return return_fields
 
 def flatten_doc(doc, return_fields):