Implements ES7 support

.travis.yml

@@ -41,6 +41,8 @@ matrix:
      python: 3.6
    - env: TOXENV=py37-dj22-postgres-elasticsearch6 INSTALL_ELASTICSEARCH6=yes
      python: 3.7
+   - env: TOXENV=py37-dj22-postgres-elasticsearch7 INSTALL_ELASTICSEARCH7=yes
+     python: 3.7  
   allow_failures:
     # Ignore failures on Elasticsearch tests because ES on Travis is intermittently flaky
     - env: TOXENV=py36-dj20-sqlite-elasticsearch2 INSTALL_ELASTICSEARCH2=yes
@@ -49,6 +51,7 @@ matrix:
     - env: TOXENV=py36-dj20-postgres-elasticsearch6 INSTALL_ELASTICSEARCH6=yes
     - env: TOXENV=py36-dj21-postgres-elasticsearch6 INSTALL_ELASTICSEARCH6=yes
     - env: TOXENV=py37-dj22-postgres-elasticsearch6 INSTALL_ELASTICSEARCH6=yes
+    - env: TOXENV=py37-dj22-postgres-elasticsearch7 INSTALL_ELASTICSEARCH7=yes
     # allow failures against Django 2.2.x stable branch
     - env: TOXENV=py37-dj22stable-postgres-noelasticsearch
     # allow failures against Django master
@@ -67,6 +70,7 @@ install:
   - 'if [[ -n "$INSTALL_ELASTICSEARCH2" ]]; then ./scripts/travis/install_elasticsearch2.sh; fi'
   - 'if [[ -n "$INSTALL_ELASTICSEARCH5" ]]; then ./scripts/travis/install_elasticsearch5.sh; fi'
   - 'if [[ -n "$INSTALL_ELASTICSEARCH6" ]]; then ./scripts/travis/install_elasticsearch6.sh; fi'
+  - 'if [[ -n "$INSTALL_ELASTICSEARCH7" ]]; then ./scripts/travis/install_elasticsearch7.sh; fi'
 
 # Pre-test configuration
 before_script:

docs/topics/search/backends.rst

@@ -91,14 +91,16 @@ See :ref:`postgres_search` for more detail.
 Elasticsearch Backend
 ---------------------
 
-Elasticsearch versions 2, 5 and 6 are supported. Use the appropriate backend for your version:
+Elasticsearch versions 2, 5, 6 and 7 are supported. Use the appropriate backend for your version:
 
 ``wagtail.search.backends.elasticsearch2`` (Elasticsearch 2.x)
 
 ``wagtail.search.backends.elasticsearch5`` (Elasticsearch 5.x)
 
 ``wagtail.search.backends.elasticsearch6`` (Elasticsearch 6.x)
 
+``wagtail.search.backends.elasticsearch7`` (Elasticsearch 7.x)
+
 Prerequisites are the `Elasticsearch`_ service itself and, via pip, the `elasticsearch-py`_ package. The major version of the package must match the installed version of Elasticsearch:
 
 .. _Elasticsearch: https://www.elastic.co/downloads/elasticsearch
@@ -115,6 +117,10 @@ Prerequisites are the `Elasticsearch`_ service itself and, via pip, the `elastic
 
   pip install "elasticsearch>=6.4.0,<7.0.0"  # for Elasticsearch 6.x
 
+.. code-block:: sh
+
+  pip install "elasticsearch>=7.0.0,<8.0.0"  # for Elasticsearch 7.x  
+
 .. warning::
 
     | Version 6.3.1 of the Elasticsearch client library is incompatible with Wagtail. Use 6.4.0 or above.

runtests.py

@@ -18,6 +18,7 @@ def make_parser():
     parser.add_argument('--elasticsearch2', action='store_true')
     parser.add_argument('--elasticsearch5', action='store_true')
     parser.add_argument('--elasticsearch6', action='store_true')
+    parser.add_argument('--elasticsearch7', action='store_true')
     parser.add_argument('--bench', action='store_true')
     return parser
 
@@ -57,6 +58,9 @@ def runtests():
     elif args.elasticsearch6:
         os.environ.setdefault('ELASTICSEARCH_URL', 'http://localhost:9200')
         os.environ.setdefault('ELASTICSEARCH_VERSION', '6')
+    elif args.elasticsearch7:
+        os.environ.setdefault('ELASTICSEARCH_URL', 'http://localhost:9200')
+        os.environ.setdefault('ELASTICSEARCH_VERSION', '7')
 
     elif 'ELASTICSEARCH_URL' in os.environ:
         # forcibly delete the ELASTICSEARCH_URL setting to skip those tests

scripts/travis/install_elasticsearch7.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.4.0-amd64.deb && sudo dpkg -i --force-confnew elasticsearch-7.4.0-amd64.deb && sudo sed -i.old 's/-Xms1g/-Xms128m/' /etc/elasticsearch/jvm.options && sudo sed -i.old 's/-Xmx1g/-Xmx128m/' /etc/elasticsearch/jvm.options && echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlog4j.skipJansi=true\n-server\n' | sudo tee -a /etc/elasticsearch/jvm.options && sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch && sudo service elasticsearch restart

tox.ini

@@ -2,7 +2,7 @@
 skipsdist = True
 usedevelop = True
 
-envlist = py{35,36,37}-dj{20,21,22,master}-{sqlite,postgres,mysql,mssql}-{elasticsearch6,elasticsearch5,elasticsearch2,noelasticsearch},
+envlist = py{35,36,37}-dj{20,21,22,master}-{sqlite,postgres,mysql,mssql}-{elasticsearch7,elasticsearch6,elasticsearch5,elasticsearch2,noelasticsearch},
 
 [flake8]
 # D100: Missing docstring in public module
@@ -30,6 +30,7 @@ commands =
     elasticsearch2: coverage run runtests.py wagtail.search wagtail.documents wagtail.images --elasticsearch2
     elasticsearch5: coverage run runtests.py wagtail.search wagtail.documents wagtail.images --elasticsearch5
     elasticsearch6: coverage run runtests.py wagtail.search wagtail.documents wagtail.images --elasticsearch6
+    elasticsearch7: coverage run runtests.py wagtail.search wagtail.documents wagtail.images --elasticsearch7
     noelasticsearch: coverage run runtests.py {posargs}
 
 basepython =
@@ -56,6 +57,8 @@ deps =
     elasticsearch5: certifi
     elasticsearch6: elasticsearch>=6.4.0,<7
     elasticsearch6: certifi
+    elasticsearch7: elasticsearch>=7,<8
+    elasticsearch7: certifi
 
 setenv =
     postgres: DATABASE_ENGINE=django.db.backends.postgresql

wagtail/search/backends/elasticsearch7.py

@@ -0,0 +1,98 @@
+from elasticsearch import NotFoundError
+from elasticsearch.helpers import bulk
+
+from wagtail.search.backends.elasticsearch2 import ElasticsearchAutocompleteQueryCompilerImpl
+from wagtail.search.backends.elasticsearch6 import (
+    Elasticsearch6Index, Elasticsearch6Mapping, Elasticsearch6SearchBackend,
+    Elasticsearch6SearchQueryCompiler, Elasticsearch6SearchResults)
+from wagtail.search.index import class_is_indexed
+
+
+class Elasticsearch7Mapping(Elasticsearch6Mapping):
+    def get_mapping(self):
+        mapping = super().get_mapping()
+        return mapping[self.get_document_type()]
+
+
+class Elasticsearch7Index(Elasticsearch6Index):
+    def add_model(self, model):
+        # Get mapping
+        mapping = self.mapping_class(model)
+
+        # Put mapping
+        self.es.indices.put_mapping(index=self.name, body=mapping.get_mapping())
+
+    def add_item(self, item):
+        # Make sure the object can be indexed
+        if not class_is_indexed(item.__class__):
+            return
+
+        # Get mapping
+        mapping = self.mapping_class(item.__class__)
+
+        # Add document to index
+        self.es.index(
+            self.name, mapping.get_document(item), id=mapping.get_document_id(item)
+        )
+
+    def add_items(self, model, items):
+        if not class_is_indexed(model):
+            return
+
+        # Get mapping
+        mapping = self.mapping_class(model)
+        doc_type = "_doc"
+
+        # Create list of actions
+        actions = []
+        for item in items:
+            # Create the action
+            action = {"_type": doc_type, "_id": mapping.get_document_id(item)}
+            action.update(mapping.get_document(item))
+            actions.append(action)
+
+        # Run the actions
+        bulk(self.es, actions, index=self.name)
+
+    def delete_item(self, item):
+        # Make sure the object can be indexed
+        if not class_is_indexed(item.__class__):
+            return
+
+        # Get mapping
+        mapping = self.mapping_class(item.__class__)
+
+        # Delete document
+        try:
+            self.es.delete(self.name, mapping.get_document_id(item))
+        except NotFoundError:
+            pass  # Document doesn't exist, ignore this exception
+
+
+class Elasticsearch7SearchQueryCompiler(Elasticsearch6SearchQueryCompiler):
+    mapping_class = Elasticsearch7Mapping
+
+
+class Elasticsearch7SearchResults(Elasticsearch6SearchResults):
+    pass
+
+
+class Elasticsearch7AutocompleteQueryCompiler(
+    Elasticsearch6SearchQueryCompiler, ElasticsearchAutocompleteQueryCompilerImpl
+):
+    pass
+
+
+class Elasticsearch7SearchBackend(Elasticsearch6SearchBackend):
+    mapping_class = Elasticsearch7Mapping
+    index_class = Elasticsearch7Index
+    query_compiler_class = Elasticsearch7SearchQueryCompiler
+    autocomplete_query_compiler_class = Elasticsearch7AutocompleteQueryCompiler
+    results_class = Elasticsearch7SearchResults
+
+    def __init__(self, params):
+        self.settings["settings"]["index"] = {"max_ngram_diff": 12}
+        super().__init__(params)
+
+
+SearchBackend = Elasticsearch7SearchBackend