global: ES7 support

.travis.yml

@@ -30,16 +30,17 @@ env:
     - ES2_DOWNLOAD_URL="https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.4.2/elasticsearch-2.4.2.tar.gz"
     - ES5_DOWNLOAD_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.4.tar.gz"
     - ES6_DOWNLOAD_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.0.tar.gz"
+    - ES7_DOWNLOAD_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.0.1-linux-x86_64.tar.gz"
     - ES_HOST=127.0.0.1
     - SQLALCHEMY_DATABASE_URI="postgresql+psycopg2://postgres@localhost:5432/invenio"
   matrix:
     - REQUIREMENTS=lowest EXTRAS=all,elasticsearch2 ES_URL=$ES2_DOWNLOAD_URL
     - REQUIREMENTS=lowest EXTRAS=all,elasticsearch5 ES_URL=$ES5_DOWNLOAD_URL
-    - REQUIREMENTS=release EXTRAS=all,elasticsearch2 ES_URL=$ES2_DOWNLOAD_URL DEPLOY=true
-    - REQUIREMENTS=release EXTRAS=all,elasticsearch5 ES_URL=$ES5_DOWNLOAD_URL
+    - REQUIREMENTS=release EXTRAS=all,elasticsearch5 ES_URL=$ES5_DOWNLOAD_URL DEPLOY=true
     - REQUIREMENTS=release EXTRAS=all,elasticsearch6 ES_URL=$ES6_DOWNLOAD_URL
     - REQUIREMENTS=devel EXTRAS=all,elasticsearch5 ES_URL=$ES5_DOWNLOAD_URL
     - REQUIREMENTS=devel EXTRAS=all,elasticsearch6 ES_URL=$ES6_DOWNLOAD_URL
+    - REQUIREMENTS=devel EXTRAS=all,elasticsearch7 ES_URL=$ES7_DOWNLOAD_URL
 
 jdk:
   - "oraclejdk8"
@@ -53,6 +54,7 @@ matrix:
   allow_failures:
     - env: REQUIREMENTS=devel EXTRAS=all,elasticsearch5 ES_URL=$ES5_DOWNLOAD_URL
     - env: REQUIREMENTS=devel EXTRAS=all,elasticsearch6 ES_URL=$ES6_DOWNLOAD_URL
+    - env: REQUIREMENTS=devel EXTRAS=all,elasticsearch7 ES_URL=$ES7_DOWNLOAD_URL
 
 before_install:
   - "mkdir /tmp/elasticsearch"

examples/app-fixtures.sh

@@ -13,9 +13,4 @@ export FLASK_APP=app.py
 
 # Create the user
 flask users create -a info@inveniosoftware.org --password 123456
-
-# Upload sample records
-echo '{"title": "Public", "body": "test 1", "public": 1}' | flask index put \
-    demo-default-v1.0.0 example
-echo '{"title": "Private", "body": "test 2", "public": 0}' | flask index put \
-    demo-default-v1.0.0 example
+flask fixtures

examples/app-teardown.sh

@@ -15,4 +15,4 @@ export FLASK_APP=app.py
 [ -e "$DIR/instance" ] && rm -Rf $DIR/instance
 
 # Delete indices
-flask index delete demo --yes-i-know
+flask index destroy --yes-i-know

examples/app.py

@@ -56,7 +56,7 @@
 from invenio_accounts.views import blueprint
 from invenio_db import InvenioDB
 
-from invenio_search import InvenioSearch, RecordsSearch
+from invenio_search import InvenioSearch, RecordsSearch, current_search_client
 
 # Create Flask application
 app = Flask(__name__)
@@ -82,24 +82,41 @@
 search.register_mappings('demo', 'data')
 
 
+@app.cli.command()
+def fixtures():
+    """Example fixtures."""
+    # Index sample records
+    current_search_client.index(
+        index='demo-default-v1.0.0',
+        body={'title': 'Public', 'body': 'test 1', 'public': 1},
+        doc_type='example' if ES_VERSION[0] < 7 else '_doc'
+    )
+    current_search_client.index(
+        index='demo-default-v1.0.0',
+        body={'title': 'Private', 'body': 'test 2', 'public': 0},
+        doc_type='example' if ES_VERSION[0] < 7 else '_doc'
+    )
+
+
 class ExampleSearch(RecordsSearch):
     """Example search class."""
 
     class Meta:
         """Configuration for ``RecordsSearch`` class."""
 
         index = 'demo'
-        doc_types = ['example']
         fields = ('*', )
         facets = {}
 
     def __init__(self, **kwargs):
         """Initialize instance."""
         super(ExampleSearch, self).__init__(**kwargs)
         if not current_user.is_authenticated:
-            self.query = self.query._proxied & Q(
-                Bool(filter=[Q('term', public=1)])
-            )
+            if self.query._proxied:
+                self.query = self.query._proxied & Q(
+                    Bool(filter=[Q('term', public=1)]))
+            else:
+                self.query = Q(Bool(filter=[Q('term', public=1)]))
 
 
 @app.route('/', methods=['GET', 'POST'])

examples/data/v7/__init__.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2015-2019 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.

examples/data/v7/demo/authorities/authority-v1.0.0.json

@@ -0,0 +1,10 @@
+{
+  "mappings": {
+    "properties": {
+      "title": {
+        "type": "text",
+        "fielddata": true
+      }
+    }
+  }
+}

examples/data/v7/demo/bibliographic/bibliographic-v1.0.0.json

@@ -0,0 +1,10 @@
+{
+  "mappings": {
+    "properties": {
+      "title": {
+        "type": "text",
+        "fielddata": true
+      }
+    }
+  }
+}

examples/data/v7/demo/default-v1.0.0.json

@@ -0,0 +1,10 @@
+{
+  "mappings": {
+    "properties": {
+      "title": {
+        "type": "text",
+        "fielddata": true
+      }
+    }
+  }
+}

setup.py

@@ -46,6 +46,10 @@
         'elasticsearch>=6.0.0,<7.0.0',
         'elasticsearch-dsl>=6.0.0,<6.2.0',
     ],
+    'elasticsearch7': [
+        'elasticsearch>=7.0.0,<8.0.0',
+        'elasticsearch-dsl>=7.0.0,<8.0.0',
+    ],
     'records': [
         'invenio-records>=1.0.0',
     ],
@@ -55,7 +59,8 @@
 extras_require['all'] = []
 for name, reqs in extras_require.items():
     if name[0] == ':' or name in (
-            'elasticsearch2', 'elasticsearch5', 'elasticsearch6'):
+            'elasticsearch2', 'elasticsearch5', 'elasticsearch6',
+            'elasticsearch7'):
         continue
     extras_require['all'].extend(reqs)
 

tests/mock_module/mappings/v7/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2018 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Tests data."""

tests/mock_module/mappings/v7/authors/authors-v1.0.0.json

@@ -0,0 +1 @@
+{"mappings": {}}

tests/mock_module/mappings/v7/records/authorities/authority-v1.0.0.json

@@ -0,0 +1 @@
+{"mappings": {}}

tests/mock_module/mappings/v7/records/authorities/notajson

@@ -0,0 +1 @@
+testfile

tests/mock_module/mappings/v7/records/bibliographic/bibliographic-v1.0.0.json

@@ -0,0 +1 @@
+{"mappings": {}}

tests/mock_module/mappings/v7/records/default-v1.0.0.json

@@ -0,0 +1 @@
+{"mappings": {}}

tests/mock_module/templates/v7/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2018 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Mock module used to test loading of ES resources."""

tests/mock_module/templates/v7/record-view-v7.json

@@ -0,0 +1,47 @@
+{
+  "index_patterns": ["__SEARCH_INDEX_PREFIX__events-stats-record-view-*"],
+  "settings": {
+    "index": {
+      "refresh_interval": "5s"
+    }
+  },
+  "mappings": {
+    "_source": {
+      "enabled": false
+    },
+    "date_detection": false,
+    "numeric_detection": false,
+    "properties": {
+      "@timestamp": {
+        "type": "date"
+      },
+      "id": {
+        "type": "text",
+        "index": false
+      },
+      "type": {
+        "type": "text",
+        "index": false
+      },
+      "value": {
+        "type": "text",
+        "index": false
+      },
+      "labels": {
+        "type": "text",
+        "index": false
+      },
+      "country": {
+        "type": "text",
+        "index": false
+      },
+      "visitor_id": {
+        "type": "text",
+        "index": false
+      }
+    }
+  },
+  "aliases": {
+    "__SEARCH_INDEX_PREFIX__events-recordview": {}
+  }
+}

tests/test_examples_app.py

@@ -18,7 +18,7 @@
 import pytest
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def example_app():
     """Example app fixture."""
     current_dir = os.getcwd()

tests/test_query.py

@@ -13,6 +13,7 @@
 
 import hashlib
 
+from elasticsearch import VERSION as ES_VERSION
 from elasticsearch_dsl import Q, Search
 from flask import request
 
@@ -22,10 +23,16 @@
 def test_empty_query(app):
     """Test building an empty query."""
     q = RecordsSearch()
-    assert q.to_dict()['query'] == {'match_all': {}}
+    if ES_VERSION[0] >= 7:
+        q.to_dict() == {}
+    else:
+        q.to_dict() == {'query': {'match_all': {}}}
 
     q = RecordsSearch.faceted_search('')
-    assert q._s.to_dict()['query'] == {'match_all': {}}
+    if ES_VERSION[0] >= 7:
+        q._s.to_dict() == {'highlight': {'fields': {'*': {}}}}
+    else:
+        q._s.to_dict() == {'query': {'match_all': {}}}
 
     q = RecordsSearch()[10]
     assert q.to_dict()['from'] == 10