Merge branch 'release/0.2'

.travis.yml

@@ -3,10 +3,10 @@ python:
 - '3.5'
 - '3.6'
 env:
-- SOLR_VERSION=6.6.5
-- DJANGO=1.11 SOLR_VERSION=6.6.5
-- DJANGO=2.0 SOLR_VERSION=6.6.5
-- DJANGO=2.1 SOLR_VERSION=6.6.5
+- SOLR_VERSION=6.6.6
+- DJANGO=1.11 SOLR_VERSION=6.6.6
+- DJANGO=2.0 SOLR_VERSION=6.6.6
+- DJANGO=2.1 SOLR_VERSION=6.6.6
 before_install:
 - pip install --upgrade pip
 - pip install --upgrade pytest

CHANGELOG.rst

@@ -3,6 +3,19 @@
 CHANGELOG
 =========
 
+
+0.2
+---
+
+* Subquent calls to SolrQuerySet.only() now *replaces* field limit options
+  rather than adding to them.
+* New SolrQuerySet method `raw_query_parameters`
+* SolrQuerySet now has support for faceting via `facet` method to configure
+  facets on the request and `get_facets` to retrieve them from the response.
+* Update `ping` method of `parasolr.solr.admin.CoreAdmin` so that
+  a 404 response is not logged as an error.
+* Refactor `parsolr.solr` tests into submodules
+
 0.1.1
 -----
 

README.rst

@@ -24,7 +24,7 @@ configuration and indexing content.
   :target: https://parasolr.readthedocs.io/en/latest/?badge=latest
   :alt: Documentation Status
 
-.. image:: https://api.codeclimate.com/v1/badges/558e86a46c76335f6673/maintainability
+.. image:: https://api.codeclimate.com/v1/badges/73394d05decdf32f12f3/maintainability
    :target: https://codeclimate.com/github/Princeton-CDH/parasolr/maintainability
    :alt: Maintainability
 

parasolr/__init__.py

@@ -1,7 +1,7 @@
 
 default_app_config = 'parasolr.apps.ParasolConfig'
 
-__version_info__ = (0, 1, 1, None)
+__version_info__ = (0, 2, 0, None)
 
 # Dot-connect all but the last. Last is dash-connected if not None.
 __version__ = '.'.join([str(i) for i in __version_info__[:-1]])

parasolr/management/commands/solr_schema.py

@@ -50,7 +50,7 @@ def handle(self, *args, **kwargs):
                 create = True
             # otherwise, prompt the user to confirm
             else:
-                create = input('Solr core %s does not exist. Create it? (y/n)' %
+                create = input('Solr core %s does not exist. Create it? (y/n) ' %
                                solr.collection).lower() == 'y'
             if create:
                 # The error handling for ensuring there's a configuration

parasolr/query.py

@@ -16,11 +16,11 @@
 which will automatically initialize a new :class:`parasolr.django.SolrClient`
 if one is not passed in.
 """
-
+from collections import OrderedDict
 from typing import Dict, List
 
 from parasolr.solr import SolrClient
-
+from parasolr.solr.client import QueryResponse
 
 class SolrQuerySet:
     """A Solr queryset object that allows for object oriented
@@ -37,7 +37,11 @@ class SolrQuerySet:
     filter_qs = []
     field_list = []
     highlight_field = None
+    facet_field = []
+    facet_opts = {}
     highlight_opts = {}
+    raw_params = {}
+
 
     #: by default, combine search queries with AND
     default_search_operator = 'AND'
@@ -69,8 +73,8 @@ def get_results(self, **kwargs) -> List[dict]:
 
         # NOTE: django templates choke on AttrDict because it is
         # callable; using dictionary response instead
-        self._result_cache = self.solr.query(wrap=False, **query_opts)
-        return self._result_cache['response']['docs']
+        self._result_cache = self.solr.query(**query_opts)
+        return [doc.as_dict() for doc in self._result_cache.docs]
 
     def query_opts(self) -> Dict[str, str]:
         """Construct query options based on current queryset configuration.
@@ -79,6 +83,7 @@ def query_opts(self) -> Dict[str, str]:
         query_opts = {
             'start': self.start,
         }
+
         if self.filter_qs:
             query_opts['fq'] = self.filter_qs
         if self.stop:
@@ -103,20 +108,55 @@ def query_opts(self) -> Dict[str, str]:
             for key, val in self.highlight_opts.items():
                 query_opts['hl.%s' % key] = val
 
+        if self.facet_field:
+            query_opts.update({
+                'facet': True,
+                'facet.field': self.facet_field
+            })
+            for key, val in self.facet_opts.items():
+                query_opts['facet.%s' % key] = val
+
+        # include any raw query parameters
+        query_opts.update(self.raw_params)
+
         return query_opts
 
     def count(self) -> int:
         """Total number of results for the current query"""
 
         # if result cache is already populated, use it
         if self._result_cache is not None:
-            return self._result_cache['response']['numFound']
+            return self._result_cache.numFound
 
         # otherwise, query with current options but request zero rows
         # and do not populate the result cache
         query_opts = self.query_opts()
+        # setting these by dictionary assignment, because conflicting
+        # kwargs results in a Python exception
         query_opts['rows'] = 0
-        return self.solr.query(**query_opts, wrap=False)['response']['numFound']
+        query_opts['facet'] = False
+        query_opts['hl'] = False
+        return self.solr.query(**query_opts).numFound
+
+    def get_facets(self) -> Dict[str, int]:
+        """Return a dictionary of facets and their values and
+        counts as key/value pairs.
+        """
+        if self._result_cache is not None:
+            # wrap to process facets and return as dictionary
+            # for Django template support
+            qr = QueryResponse(self._result_cache)
+            # NOTE: using dictionary syntax preserves OrderedDict
+            return qr.facet_counts['facet_fields']
+        # since we just want a dictionary of facet fields, don't populate
+        # the result cache, no rows needed
+
+        query_opts = self.query_opts()
+        query_opts['rows'] = 0
+        query_opts['hl'] = False
+        # setting these by dictionary assignment, because conflicting
+        # kwargs results in a Python exception
+        return self.solr.query(**query_opts).facet_counts['facet_fields']
 
     @staticmethod
     def _lookup_to_filter(key, value) -> str:
@@ -127,7 +167,7 @@ def _lookup_to_filter(key, value) -> str:
         # such as __in=[a, b, c] or __range=(start, end)
         return '%s:%s' % (key, value)
 
-    def filter(self, *args, **kwargs):
+    def filter(self, *args, **kwargs) -> 'SolrQuerySet':
         """
         Return a new SolrQuerySet with Solr filter queries added.
         Multiple filters can be combined either in a single
@@ -153,7 +193,32 @@ def filter(self, *args, **kwargs):
 
         return qs_copy
 
-    def search(self, *args, **kwargs):
+    def facet(self, *args: str, **kwargs) -> 'SolrQuerySet':
+        """
+        Request facets for specified fields. Returns a new SolrQuerySet
+        with Solr faceting enabled and facet.field parameter set. Does not
+        support ranged faceting.
+
+        Subsequent calls will reset the facet.field to the last set of
+        args in the chain.
+
+        For example::
+
+            qs = queryset.facet('person_type', 'age')
+            qs = qs.facet('item_type')
+
+        would result in `item_type` being the only facet field.
+        """
+        qs_copy = self._clone()
+
+        # cast args tuple to list for consistency with other iterable fields
+        qs_copy.facet_field = list(args)
+        # add other kwargs to be prefixed in query_opts
+        qs_copy.facet_opts.update(kwargs)
+
+        return qs_copy
+
+    def search(self, *args, **kwargs) -> 'SolrQuerySet':
         """
         Return a new SolrQuerySet with search queries added. All
         queries will combined with the default search operator when
@@ -168,7 +233,7 @@ def search(self, *args, **kwargs):
 
         return qs_copy
 
-    def order_by(self, *args):
+    def order_by(self, *args) -> 'SolrQuerySet':
         """Apply sort options to the queryset by field name. If the field
         name starts with -, sort is descending; otherwise ascending."""
         qs_copy = self._clone()
@@ -182,7 +247,7 @@ def order_by(self, *args):
 
         return qs_copy
 
-    def query(self, **kwargs):
+    def query(self, **kwargs) -> 'SolrQuerySet':
         """Return a new SolrQuerySet with the results populated from Solr.
         Any options passed in via keyword arguments take precedence
         over query options on the queryset.
@@ -191,22 +256,24 @@ def query(self, **kwargs):
         qs_copy.get_results(**kwargs)
         return qs_copy
 
-    def only(self, *args, **kwargs):
+    def only(self, *args, **kwargs) -> 'SolrQuerySet':
         """Use field limit option to return only the specified fields.
-        Optionally provide aliases for them in the return. Example::
+        Optionally provide aliases for them in the return. Subsequent
+        calls will *replace* any previous field limits. Example::
 
             queryset.only('title', 'author', 'date')
             queryset.only('title:title_t', 'date:pubyear_i')
 
         """
         qs_copy = self._clone()
-        qs_copy.field_list.extend(args)
+        # *replace* any existing field list with the current values
+        qs_copy.field_list = list(args)
         for key, value in kwargs.items():
             qs_copy.field_list.append('%s:%s' % (key, value))
 
         return qs_copy
 
-    def highlight(self, field: str, **kwargs):
+    def highlight(self, field: str, **kwargs) -> 'SolrQuerySet':
         """"Configure highlighting. Takes arbitrary Solr highlight
         parameters and adds the `hl.` prefix to them.  Example use::
 
@@ -217,24 +284,32 @@ def highlight(self, field: str, **kwargs):
         qs_copy.highlight_opts = kwargs
         return qs_copy
 
+    def raw_query_parameters(self, **kwargs) -> 'SolrQuerySet':
+        """Add abritrary raw parameters to be included in the query
+        request, e.g. for variables referenced in join or field queries.
+        Analogous to the input of the same name in the Solr web interface."""
+        qs_copy = self._clone()
+        qs_copy.raw_params.update(kwargs)
+        return qs_copy
+
     def get_highlighting(self):
         """Return the highlighting portion of the Solr response."""
         if not self._result_cache:
             self.get_results()
         return self._result_cache.get('highlighting', {})
 
-    def all(self):
+    def all(self) -> 'SolrQuerySet':
         """Return a new queryset that is a copy of the current one."""
         return self._clone()
 
-    def none(self):
+    def none(self) -> 'SolrQuerySet':
         """Return an empty result list."""
         qs_copy = self._clone()
         # replace any search queries with this to find not anything
         qs_copy.search_qs = ['NOT *:*']
         return qs_copy
 
-    def _clone(self):
+    def _clone(self) -> 'SolrQuerySet':
         """
         Return a copy of the current QuerySet for modification via
         filters.
@@ -247,17 +322,21 @@ def _clone(self):
         qs_copy.stop = self.stop
         qs_copy.highlight_field = self.highlight_field
 
-        # set copies of list attributes
+        # set copies of list and dict attributes
         qs_copy.search_qs = list(self.search_qs)
         qs_copy.filter_qs = list(self.filter_qs)
         qs_copy.sort_options = list(self.sort_options)
         qs_copy.field_list = list(self.field_list)
         qs_copy.highlight_opts = dict(self.highlight_opts)
+        qs_copy.raw_params = dict(self.raw_params)
+        qs_copy.facet_field = list(self.facet_field)
+        qs_copy.facet_opts = dict(self.facet_opts)
+
 
         return qs_copy
 
     def set_limits(self, start, stop):
-        """Return a subsection of the results, to support slicing."""
+        """Set limits to get a subsection of the results, to support slicing."""
         if start is None:
             start = 0
         self.start = start
@@ -287,7 +366,7 @@ def __getitem__(self, k):
         # if the result cache is already populated,
         # return the requested index or slice
         if self._result_cache is not None:
-            return self._result_cache['response']['docs'][k]
+            return self._result_cache.docs[k]
 
         qs_copy = self._clone()
 

parasolr/solr/admin.py

@@ -4,7 +4,7 @@
 from attrdict import AttrDict
 import requests
 
-from parasolr.solr.client import ClientBase
+from parasolr.solr.base import ClientBase
 
 
 class CoreAdmin(ClientBase):
@@ -79,8 +79,9 @@ def ping(self, core: str) -> bool:
 
         """
         ping_url = '/'.join([self.solr_url.rstrip('/'), core, 'admin', 'ping'])
-        response = self.make_request('get', ping_url)
-        # ping returns 404 if core does not exist (make request returns None)
-
+        # ping returns 404 if core does not exist, but that's ok here
+        allowed_responses = [requests.codes.ok, requests.codes.not_found]
+        response = self.make_request('get', ping_url,
+                                     allowed_responses=allowed_responses)
         # return True if response is valid and status is OK
         return response and response.status == 'OK'