Merge 118a142 into 6a58327

.travis.yml

@@ -36,33 +36,50 @@ cache:
   - pip
 
 services:
-  - elasticsearch
   - redis
   - rabbitmq
 
 env:
-  - REQUIREMENTS=lowest BROKER_URL=redis://localhost:6379/0
-  - REQUIREMENTS=lowest BROKER_URL=amqp://localhost:5672//
-  - REQUIREMENTS=release BROKER_URL=redis://localhost:6379/0
-  - REQUIREMENTS=release BROKER_URL=amqp://localhost:5672// DEPLOY=true
-  - REQUIREMENTS=devel BROKER_URL=redis://localhost:6379/0
-  - REQUIREMENTS=devel BROKER_URL=amqp://localhost:5672//
+  global:
+    - ES2_DOWNLOAD_URL="https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.4.2/elasticsearch-2.4.2.tar.gz"
+    - ES5_DOWNLOAD_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.4.tar.gz"
+    - ES_HOST=127.0.0.1
+  matrix:
+    # ES 5: Redis vs RabbitMQ
+    - REQUIREMENTS=lowest EXTRAS=all,elasticsearch5 BROKER_URL=redis://localhost:6379/0 ES_URL=$ES5_DOWNLOAD_URL
+    - REQUIREMENTS=lowest EXTRAS=all,elasticsearch5 BROKER_URL=amqp://localhost:5672// ES_URL=$ES5_DOWNLOAD_URL
+    - REQUIREMENTS=release EXTRAS=all,elasticsearch5 BROKER_URL=redis://localhost:6379/0 ES_URL=$ES5_DOWNLOAD_URL
+    - REQUIREMENTS=release EXTRAS=all,elasticsearch5 BROKER_URL=amqp://localhost:5672// ES_URL=$ES5_DOWNLOAD_URL DEPLOY=true
+    - REQUIREMENTS=devel EXTRAS=all,elasticsearch5 BROKER_URL=redis://localhost:6379/0 ES_URL=$ES5_DOWNLOAD_URL
+    - REQUIREMENTS=devel EXTRAS=all,elasticsearch5 BROKER_URL=amqp://localhost:5672// ES_URL=$ES5_DOWNLOAD_URL
+    # ES 2: Redis only
+    - REQUIREMENTS=lowest EXTRAS=all,elasticsearch2 BROKER_URL=redis://localhost:6379/0 ES_URL=$ES2_DOWNLOAD_URL
+    - REQUIREMENTS=release EXTRAS=all,elasticsearch2 BROKER_URL=redis://localhost:6379/0 ES_URL=$ES2_DOWNLOAD_URL
+    - REQUIREMENTS=devel EXTRAS=all,elasticsearch2 BROKER_URL=redis://localhost:6379/0 ES_URL=$ES2_DOWNLOAD_URL
+
+jdk:
+  - "oraclejdk8"
 
 python:
   - "2.7"
   - "3.5"
 
 before_install:
   - "nvm install 6; nvm use 6"
+  - "mkdir /tmp/elasticsearch"
+  - "wget -O - $ES_URL | tar xz --directory=/tmp/elasticsearch --strip-components=1"
+  - "/tmp/elasticsearch/bin/elasticsearch &"
+  - "sleep 20"
   - "travis_retry pip install --upgrade pip setuptools py"
   - "travis_retry pip install twine wheel coveralls requirements-builder"
-  - "requirements-builder -e all --level=min setup.py > .travis-lowest-requirements.txt"
-  - "requirements-builder -e all --level=pypi setup.py > .travis-release-requirements.txt"
-  - "requirements-builder -e all --level=dev --req requirements-devel.txt setup.py > .travis-devel-requirements.txt"
+  - "requirements-builder -e $EXTRAS --level=min setup.py > .travis-lowest-requirements.txt"
+  - "requirements-builder -e $EXTRAS --level=pypi setup.py > .travis-release-requirements.txt"
+  - "requirements-builder -e $EXTRAS --level=dev --req requirements-devel.txt setup.py > .travis-devel-requirements.txt"
+
 
 install:
   - "travis_retry pip install -r .travis-${REQUIREMENTS}-requirements.txt"
-  - "travis_retry pip install -e .[all]"
+  - "travis_retry pip install -e .[$EXTRAS]"
 
 script:
   - "./run-tests.sh"
@@ -74,8 +91,7 @@ deploy:
   provider: pypi
   user: inveniosoftware
   password:
-    secure: TODO:PYPISECUREHASH
-  distributions: "compile_catalog sdist bdist_wheel"
+    secure: "IJF9L5eR24EED8eslPKR0LacibaG2eUQZT8lTurI/xp15D9NLqfoBotyTG7lLBWmI7sRQwL2UyYeZ7p2Ck0K1HAOtAC+dJUxTa89dFk3FrJI58p+hc3YOxk5MsJaO+q+Hpd/drf8r3ETEz0MJAMHB1oYCUnQRIc2XBRpCcq3XukMVMv/UWhSubOFN9kLQJCwbA+6wjrMpmeNfSQ2+yVKpolJiTcMpCPFbEAQ2Gr/ifA6aOFczku3d11fG73gNu4zqTJCdnjF7n1QYaQzkR3kSTSgBPHocslKdFiJfrzJHqVreniqbRmO3fUI3XSCejo2cXqccKTGnPbNDz+lM4fKxlaK4lPZRfXSCpQFOlzMPkmcvy2agpzKCSyVAl6+/ukT9XDUDFl/WYjRx4g0HhzzcIi6Lf24aviM4cJrwsUPeZcZGQALiFZX5pTTPy0GzlVFUKGeylk6J3TqyyUAq0pDPdzHpcFMWpgVfX4PBqsbWvORO7p9mxpOtUSn1AVtqKSXRoV599HY7IFudf0nakir2h9A7Stv1nhmdatJphBxQDNxkG1fOK2flAeSr344HWjTVRY4i0zS4RRJcb0O7O88dDFSnQ3jMcwWu2sElzTGqQ6xxjJS9puZvuXV9XI0nHDkKiBC1uHeE2Q5JudG9XxAjfT/Fk2L4n6ZeQrsZUhQ5kA="
   on:
     tags: true
     python: "2.7"

examples/data/v5/testrecords/testrecord-v1.0.0.json

@@ -0,0 +1,27 @@
+{
+  "mappings": {
+    "testrecord-v1.0.0": {
+      "properties": {
+        "control_number": {
+          "type": "keyword"
+        },
+        "description": {
+          "type": "text"
+        },
+        "participants": {
+          "type": "long"
+        },
+        "title": {
+          "type": "text",
+          "copy_to": "suggest_title"
+        },
+        "type": {
+          "type": "keyword"
+        },
+        "suggest_title": {
+          "type": "completion"
+        }
+      }
+    }
+  }
+}

invenio_indexer/__init__.py

@@ -102,11 +102,22 @@
 Bulk indexing
 -------------
 If you have many records to index, bulk indexing is far superior in speed to
-single record indexing.
+single record indexing. Bulk indexing requires the existence of a queue on your
+broker, so since this is the very first time we send any records for bulk
+indexing, we will have to create this queue:
+
+>>> from celery.messaging import establish_connection
+>>> queue = app.config['INDEXER_MQ_QUEUE']
+>>> with establish_connection() as conn:
+...     queue(conn).declare()
+'indexer'
+
+
+We can now send a record for bulk indexing:
 
 >>> indexer.bulk_index([str(r.id)])
 
-Above will send the record id to a queue on your broker and wait for the bulk
+Above will send the record id to the queue on your broker and wait for the bulk
 indexer to execute. This is normally done in the background by a Celery task
 which can be started from the command line like e.g.:
 

invenio_indexer/utils.py

@@ -25,39 +25,9 @@
 """Utility functions for data processing."""
 
 from flask import current_app
-from invenio_records.models import RecordMetadata
 from invenio_search import current_search
 from invenio_search.utils import schema_to_index
 
-from .api import RecordIndexer
-
-
-def process_models_committed_signal(sender, changes):
-    """Handle the indexing of record metadata.
-
-    :param sender: The signal sender.
-    :param changes: The changes sent: a list of tuple (record, action).
-    """
-    record_indexer = RecordIndexer()
-    op_map = {
-        'insert': 'index',
-        'update': 'index',
-        'delete': 'delete',
-    }
-    with record_indexer.create_producer() as producer:
-        for obj, change in changes:
-            if isinstance(obj, RecordMetadata):
-                if change in op_map:
-                    index, doc_type = record_indexer.record_to_index(
-                        obj.json or {}
-                    )
-                    producer.publish(dict(
-                        op=op_map[change],
-                        id=str(obj.id),
-                        index=index,
-                        doc_type=doc_type,
-                    ))
-
 
 def default_record_to_index(record):
     """Get index/doc_type given a record.

setup.py

@@ -34,7 +34,7 @@
 tests_require = [
     'check-manifest>=0.25',
     'coverage>=4.0',
-    'invenio-db[versioning]>=1.0.0a8',
+    'invenio-db[versioning]>=1.0.0b9',
     'isort>=4.2.2',
     'mock>=1.3.0',
     'pydocstyle>=1.0.0',
@@ -45,30 +45,39 @@
     'redis>=2.10.0',
 ]
 
+invenio_search_version = '1.0.0b3'
+
 extras_require = {
     'docs:python_version=="2.7"': [
         'celery>=3.1.16',
     ],
     'docs': [
         'Sphinx>=1.5.1,<1.6',
     ],
+    'elasticsearch2': [
+        'invenio-search[elasticsearch2]>={}'.format(invenio_search_version),
+    ],
+    'elasticsearch5': [
+        'invenio-search[elasticsearch5]>={}'.format(invenio_search_version),
+    ],
     'tests': tests_require,
 }
 
 extras_require['all'] = []
-for reqs in extras_require.values():
+for name, reqs in extras_require.items():
+    if name[0] == ':' or name in ('elasticsearch2', 'elasticsearch5'):
+        continue
     extras_require['all'].extend(reqs)
 
 setup_requires = [
     'pytest-runner>=2.6.2',
 ]
 
 install_requires = [
-    'Flask-CeleryExt>=0.2.2',
     'Flask>=0.11.1',
-    'invenio-pidstore>=1.0.0b1',
-    'invenio-records>=1.0.0a8',
-    'invenio-search>=1.0.0a7',
+    'Flask-CeleryExt>=0.3.0',
+    'invenio-pidstore>=1.0.0b2',
+    'invenio-records>=1.0.0b4',
     'pytz>=2016.4',
 ]
 

tests/conftest.py

@@ -60,7 +60,7 @@ def base_app(request):
         INDEXER_DEFAULT_DOC_TYPE='default-v1.0.0',
         SQLALCHEMY_DATABASE_URI=os.environ.get(
             'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'),
-        SQLALCHEMY_TRACK_MODIFICATIONS=True,
+        SQLALCHEMY_TRACK_MODIFICATIONS=False,
         TESTING=True,
     )
     FlaskCeleryExt(app)

tests/data/v2/__init__.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2017 CERN.
+#
+# Invenio is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+"""Elasticsearch v2 mappings."""

tests/data/v5/__init__.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2017 CERN.
+#
+# Invenio is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+"""Elasticsearch v5 mappings."""

tests/data/v5/records/authorities/authority-v1.0.0.json

@@ -0,0 +1 @@
+{"mappings": {}}

tests/data/v5/records/bibliographic/bibliographic-v1.0.0.json

@@ -0,0 +1 @@
+{"mappings": {}}

tests/data/v5/records/default-v1.0.0.json

@@ -0,0 +1 @@
+{"mappings": {}}