Solr6 template upgrade

.travis.yml

@@ -1,4 +1,5 @@
-sudo: true
+sudo: required
+dist: trusty
 language: python
 python:
     - 2.7
@@ -11,6 +12,8 @@ cache:
     pip: true
     directories:
         - $HOME/download-cache
+jdk:
+    - oraclejdk8
 
 addons:
     apt_packages:
@@ -59,6 +62,7 @@ env:
         - DJANGO_VERSION=">=1.8,<1.9" VERSION_ES=">=2.0.0,<3.0.0"
         - DJANGO_VERSION=">=1.9,<1.10" VERSION_ES=">=2.0.0,<3.0.0"
         - DJANGO_VERSION=">=1.10,<1.11" VERSION_ES=">=2.0.0,<3.0.0"
+        - DJANGO_VERSION=">=1.11,<1.12" VERSION_ES=">=2.0.0,<3.0.0"
 
 matrix:
     allow_failures:

haystack/__init__.py

@@ -54,7 +54,8 @@
 # DEBUG = True.
 def reset_search_queries(**kwargs):
     for conn in connections.all():
-        conn.reset_queries()
+        if conn:
+            conn.reset_queries()
 
 
 if settings.DEBUG:

haystack/backends/solr_backend.py

@@ -45,6 +45,8 @@ def __init__(self, connection_alias, **connection_options):
         if 'URL' not in connection_options:
             raise ImproperlyConfigured("You must specify a 'URL' in your settings for connection '%s'." % connection_alias)
 
+        self.collate = connection_options.get('COLLATE_SPELLING', True)
+
         self.conn = Solr(connection_options['URL'], timeout=self.timeout,
                          **connection_options.get('KWARGS', {}))
         self.log = logging.getLogger('haystack')
@@ -151,7 +153,7 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of
                             narrow_queries=None, spelling_query=None,
                             within=None, dwithin=None, distance_point=None,
                             models=None, limit_to_registered_models=None,
-                            result_class=None, stats=None,
+                            result_class=None, stats=None, collate=None,
                             **extra_kwargs):
         kwargs = {'fl': '* score'}
 
@@ -201,9 +203,11 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of
                     for key in highlight.keys()
                 })
 
+        if collate is None:
+            collate = self.collate
         if self.include_spelling is True:
             kwargs['spellcheck'] = 'true'
-            kwargs['spellcheck.collate'] = 'true'
+            kwargs['spellcheck.collate'] = str(collate).lower()
             kwargs['spellcheck.count'] = 1
 
             if spelling_query:
@@ -389,13 +393,38 @@ def _process_results(self, raw_results, highlight=False, result_class=None, dist
                                                         facets[key][facet_field][1::2]))
 
         if self.include_spelling and hasattr(raw_results, 'spellcheck'):
-            # Solr 5+ changed the JSON response format so the suggestions will be key-value mapped rather
-            # than simply paired elements in a list, which is a nice improvement but incompatible with
-            # Solr 4: https://issues.apache.org/jira/browse/SOLR-3029
-            if len(raw_results.spellcheck.get('collations', [])):
-                spelling_suggestion = raw_results.spellcheck['collations'][-1]
-            elif len(raw_results.spellcheck.get('suggestions', [])):
-                spelling_suggestion = raw_results.spellcheck['suggestions'][-1]
+            # There are many different formats for Legacy, 6.4, and 6.5
+            # e.g. https://issues.apache.org/jira/browse/SOLR-3029
+            collations = raw_results.spellcheck.get('collations', [])
+            suggestions = raw_results.spellcheck.get('suggestions', [])
+            if len(collations):
+                #Handle sol6.5 collation format
+                if isinstance(collations, dict):
+                    spelling_suggestions= [col['collationQuery'] for col in collations.values()]  #aggregate for future use in multi suggestion response
+                #Legacy Legacy & 6.4 handling
+                else:
+                    if isinstance(collations[1], dict):  #Solr6.4
+                        spelling_suggestions = [item["collationQuery"] for item in collations if isinstance(item,dict)]  #aggregate for future use in multi suggestion response
+                    else:  #Legacy Solr format
+                        spelling_suggestions=collations[-1]
+
+                spelling_suggestion = spelling_suggestions[-1]  #Keep current method of returning single value
+            elif len(suggestions):
+                #Handle sol6.5 suggestion format
+                if isinstance(suggestions, dict):
+                    for word,sug in suggestions.items():
+                        spelling_suggestions = [item["word"] for item in sug['suggestion']]  #aggregate for future use in multi suggestion response
+                #Legacy Legacy & 6.4 handling
+                else:
+                    spelling_suggestions = []
+                    if isinstance(suggestions[1], dict):  #Solr6.4
+                        for item in suggestions:
+                            if isinstance(item, dict):
+                                spelling_suggestions += [subitem["word"] for subitem in item['suggestion']]
+                    else:  #Legacy Solr
+                        spelling_suggestions=suggestions[-1]
+
+                spelling_suggestion = spelling_suggestions[-1]  #Keep current method of returning single value
 
             assert spelling_suggestion is None or isinstance(spelling_suggestion, six.string_types)
 
@@ -722,6 +751,7 @@ def run(self, spelling_query=None, **kwargs):
             search_kwargs.update(kwargs)
 
         results = self.backend.search(final_query, **search_kwargs)
+
         self._results = results.get('results', [])
         self._hit_count = results.get('hits', 0)
         self._facet_counts = self.post_process_facets(results)

haystack/constants.py

@@ -10,6 +10,7 @@
 ID = getattr(settings, 'HAYSTACK_ID_FIELD', 'id')
 DJANGO_CT = getattr(settings, 'HAYSTACK_DJANGO_CT_FIELD', 'django_ct')
 DJANGO_ID = getattr(settings, 'HAYSTACK_DJANGO_ID_FIELD', 'django_id')
+HAYSTACK_DOCUMENT_FIELD = getattr(settings, 'HAYSTACK_DOCUMENT_FIELD', 'text')
 
 # Default operator. Valid options are AND/OR.
 DEFAULT_OPERATOR = getattr(settings, 'HAYSTACK_DEFAULT_OPERATOR', 'AND')

haystack/management/commands/build_solr_schema.py

@@ -3,15 +3,21 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from django.core.exceptions import ImproperlyConfigured
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand,CommandError
 from django.template import Context, loader
+from django.conf import settings
 
 from haystack import connections, connection_router, constants
 from haystack.backends.solr_backend import SolrSearchBackend
 
-
+import pysolr
+import os
+import traceback
+import requests
 class Command(BaseCommand):
-    help = "Generates a Solr schema that reflects the indexes."
+    help = "Generates a Solr schema that reflects the indexes using templates under a django template dir 'search_configuration/*.xml'"
+    schema_template_loc = 'search_configuration/schema.xml'
+    solrcfg_template_loc = 'search_configuration/solrconfig.xml'
 
     def add_arguments(self, parser):
         parser.add_argument(
@@ -22,15 +28,66 @@ def add_arguments(self, parser):
             "-u", "--using", default=constants.DEFAULT_ALIAS,
             help='If provided, chooses a connection to work with.'
         )
+        parser.add_argument(
+            "-c", "--configure_dir",
+            help='If provided, attempts to configure a core located in the given directory by removing the managed-schema.xml(renaming), configuring the core to use a classic (non-dynamic) schema, and generating the schema.xml from the template provided in'
+        )
+        parser.add_argument(
+            "-r", "--reload",
+            help='If provided, attempts to automatically reload the solr core'
+        )
+
 
     def handle(self, **options):
         """Generates a Solr schema that reflects the indexes."""
         using = options.get('using')
-        schema_xml = self.build_template(using=using)
+        if not isinstance(connections[using].get_backend(), SolrSearchBackend):
+            raise ImproperlyConfigured("'%s' isn't configured as a SolrEngine)." % connections[using].get_backend().connection_alias)
+
+        schema_xml = self.build_template(using=using,tfile=Command.schema_template_loc)
+        solrcfg_xml = self.build_template(using=using,tfile=Command.solrcfg_template_loc)
 
         if options.get('filename'):
+            self.stdout.write("Trying to write schema file located at {}".format(options.get('filename')))
             self.write_file(options.get('filename'), schema_xml)
-        else:
+            if options.get('reload'):
+                connections[using].get_backend().reload()
+
+        if options.get('configure_dir'):
+            cdir = options.get('configure_dir')
+            self.stdout.write("Trying to configure core located at {}".format(cdir))
+            if os.path.isfile(cdir+'/managed-schema'):
+                try:
+                    os.rename(cdir+'/managed-schema',cdir+'/managed-schema.old')
+                except:
+                    raise CommandError('Could not rename managed schema out of the way: {}'.format(cdir+'/managed-schema'))
+            try:
+                self.write_file(cdir+'/schema.xml', schema_xml)
+            except:
+                raise CommandError('Could not configure {}: {}'.format(cdir+'/schema.xml',traceback.format_exc()))
+
+            try:
+                self.write_file(cdir+'/solrconfig.xml',solrcfg_xml)
+            except:
+                raise CommandError('Could not configure core to use classic Schema Factory {}'.format(cdir+'/solrconfig.xml'))
+
+        if options.get('reload'):
+            core= settings.HAYSTACK_CONNECTIONS['solr']['URL'].rsplit('/',1)[-1]
+            if 'ADMIN_URL' not in settings.HAYSTACK_CONNECTIONS['solr']:
+                raise ImproperlyConfigured("'ADMIN_URL' must be specifid in the HAYSTACK_CONNECTIONS settins for the backend." )
+            if 'URL' not in settings.HAYSTACK_CONNECTIONS['solr']:
+                raise ImproperlyConfigured("'URL' to the core must be specifid in the HAYSTACK_CONNECTIONS settins for the backend.")
+            try:
+                self.stdout.write("Trying to relaod core named {}".format(core))
+                resp = requests.get(settings.HAYSTACK_CONNECTIONS['solr']['ADMIN_URL'],params="action=RELOAD&core="+core).text#TODO: Fix when pysolr passes params as request params instead of data
+                if resp.find('SolrException')!=-1:
+                    raise CommandError('Solr Exception Thrown -- Failed to reload core: {}'.format(resp))
+            except CommandError:
+                raise
+            except:
+                raise CommandError('Failed to reload core: {}'.format(traceback.format_exc()))
+
+        if  options.get('filename') is None and options.get('configure_dir') is None and options.get('reload') is None:
             self.print_stdout(schema_xml)
 
     def build_context(self, using):
@@ -42,17 +99,17 @@ def build_context(self, using):
         content_field_name, fields = backend.build_schema(
             connections[using].get_unified_index().all_searchfields()
         )
-        return Context({
+        return {
             'content_field_name': content_field_name,
             'fields': fields,
             'default_operator': constants.DEFAULT_OPERATOR,
             'ID': constants.ID,
             'DJANGO_CT': constants.DJANGO_CT,
             'DJANGO_ID': constants.DJANGO_ID,
-        })
+        }
 
-    def build_template(self, using):
-        t = loader.get_template('search_configuration/solr.xml')
+    def build_template(self, using, tfile=schema_template_loc):
+        t = loader.get_template(tfile)
         c = self.build_context(using=using)
         return t.render(c)
 
@@ -68,3 +125,4 @@ def print_stdout(self, schema_xml):
     def write_file(self, filename, schema_xml):
         with open(filename, 'w') as schema_file:
             schema_file.write(schema_xml)
+            os.fsync(schema_file.fileno())