examples: search engine integration

.editorconfig

@@ -37,6 +37,7 @@ charset = utf-8
 indent_size = 4
 # isort plugin configuration
 known_first_party = invenio_marc21
+known_third_party = dojson
 multi_line_output = 2
 default_section = THIRDPARTY
 

MANIFEST.in

@@ -41,3 +41,5 @@ recursive-include invenio_marc21 *.html
 recursive-include invenio_marc21 *.po *.pot *.mo
 recursive-include invenio_marc21 *.json
 recursive-include tests *.py
+recursive-include examples *.html
+recursive-include examples *.css

examples/app.py

@@ -23,34 +23,166 @@
 # as an Intergovernmental Organization or submit itself to any jurisdiction.
 
 
-"""Minimal Flask application example for development.
+r"""Minimal Flask application example for development.
 
-Run example development server:
+Usage:
 
-.. code-block:: console
+1. Install elasticsearch:
+
+    .. code-block:: console
+
+        wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch \
+            | apt-key add -
+        echo "deb http://packages.elastic.co/elasticsearch/2.x/debian stable \
+            main" | tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list
+        apt-get update && apt-get install -y elasticsearch default-jdk
+        service elasticsearch start
+
+2. Install invenio-marc21:
+
+    .. code-block:: console
+
+        cdvirtualenv src/invenio-marc21
+        pip install -e .[all]
+
+3. Create database and tables:
+
+    .. code-block:: console
+
+        $ cd examples
+        $ flask -a app.py db init
+        $ flask -a app.py db create
+
+You can find the database in `examples/app.db`.
+
+4. Load demo records from invenio-records (see
+invenio_records/data/marc21/bibliographic.xml):
+
+    .. code-block:: console
+
+        $ wget "https://github.com/inveniosoftware/invenio-records/raw/\
+            master/invenio_records/data/marc21/bibliographic.xml"
+        $ dojson -i bibliographic.xml -l marcxml do marc21 | \
+            flask -a app.py records create
+        $ flask -a app.py fixtures records
+
+5. Run the development server:
+
+    .. code-block:: console
+
+        flask -a app.py run -h 0.0.0.0 -p 5000
+
+
+6a. Open in a browser the page `http://0.0.0.0:5000/example/[:pid]`.
+
+   E.g. open `http://0.0.0.0:5000/example/1`
+
+
+6b. Open in a browser the page `http://0.0.0.0:5000/search?q=[:query]`.
+
+   E.g. open `http://0.0.0.0:5000/search?q=control_number:1`
 
-   $ cd examples
-   $ flask -a app.py --debug run
 """
 
 from __future__ import absolute_import, print_function
 
-from flask import Flask
+import os
+
+from flask import Blueprint, Flask, render_template, request
 from flask_babelex import Babel
 from flask_cli import FlaskCLI
-from invenio_search import InvenioSearch
+from invenio_db import InvenioDB, db
+from invenio_pidstore.minters import recid_minter
+from invenio_pidstore.models import PersistentIdentifier
+from invenio_records import InvenioRecords
+from invenio_records.models import RecordMetadata
+from invenio_records.api import Record
+from invenio_search import InvenioSearch, current_search, \
+    current_search_client, Query
+from invenio_search.utils import schema_to_index
 
 from invenio_marc21 import InvenioMARC21
 
+from invenio_indexer.api import RecordIndexer
+
+
 # Create Flask application
 app = Flask(__name__)
 app.config.update(
     ELASTICSEARCH_HOST="localhost:9200",
     SEARCH_ELASTIC_HOSTS="localhost:9200",
     SEARCH_ELASTIC_KEYWORD_MAPPING={None: ['_all']},
+    SQLALCHEMY_DATABASE_URI=os.getenv('SQLALCHEMY_DATABASE_URI',
+                                      'sqlite:///app.db'),
 )
-
+app.debug = True
 Babel(app)
 FlaskCLI(app)
+InvenioDB(app)
+InvenioRecords(app)
+search = InvenioSearch(app)
 InvenioSearch(app)
 InvenioMARC21(app)
+
+
+blueprint = Blueprint(
+    'invenio_marc21',
+    __name__,
+    template_folder='templates',
+    static_folder='static',
+)
+
+
+@app.route('/example/<string:index>', methods=['GET'])
+def example(index):
+    """Index page."""
+    pid = PersistentIdentifier.query.filter_by(id=index).one()
+    record = RecordMetadata.query.filter_by(id=pid.object_uuid).first()
+    record.json['control_number'] = pid.pid_value
+    return render_template("app/detail.html", record=record.json,
+                           title="Demosite Invenio Org")
+
+
+@app.route('/search', methods=['GET', 'POST'])
+def search():
+    """Query Elasticsearch using Invenio query syntax."""
+    index_names = current_search.mappings.keys()
+    schema = "mappings/marc21_holdings.json"
+    index, doc_type = schema_to_index(schema, index_names=index_names)
+    page = request.values.get('page', 1, type=int)
+    #  size = request.values.get('size', 1, type=int)
+    size = 1
+    query = Query(request.values.get('q', ''))[(page-1)*size:page*size]
+    response = current_search_client.search(
+        index=index,
+        doc_type=doc_type,
+        body=query.body,
+    )
+    #  return jsonify(**response)
+    record = response['hits']['hits'][0]['_source']
+    return render_template("app/detail.html", record=record,
+                           title="Demosite Invenio Org")
+
+
+@app.cli.group()
+def fixtures():
+    """Initialize example data."""
+
+
+@fixtures.command()
+def records():
+    """Load fixtures."""
+    # clean indices
+    current_search_client.indices.delete_alias(
+        '_all', '_all', ignore=[400, 404])
+    current_search_client.indices.delete('*')
+    # mint and index record
+    with db.session.begin_nested():
+        indexer = RecordIndexer()
+        for record in RecordMetadata.query.all():
+            recid_minter(record.id, record.json)
+            record = Record(record.json, model=record)
+            record['$schema'] = "mappings/marc21_holdings.json"
+            indexer.index(record)
+            db.session.merge(record.model)
+    db.session.commit()

examples/static/app/css/app.css

@@ -0,0 +1,24 @@
+.title{
+  background-color: #7775f5;
+  color: #ebebeb;
+  text-align: center;
+}
+.container{
+  display: inline-block;
+  width: 100%;
+  margin-bottom: 1em;
+}
+h2 small{
+  color: #9f9393;
+}
+.panel{
+  border-top: 1px solid;
+  color: #a69e9e;
+}
+.extra-info{
+  float: right;
+  background-image: linear-gradient(to bottom, #e8e8e8 0px, #f5f5f5 100%);
+  line-height: 1.5em;
+  padding: 1em;
+  margin: 1.5em 1em;
+}

examples/templates/app/detail.html

@@ -0,0 +1,42 @@
+{# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2016 CERN.
+#
+# Invenio is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+-#}
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="/static/app/css/app.css" />
+  </head>
+  <body>
+
+    <h1 class=title>{{ title }}</h1>
+
+    {%- extends "app/detail_base.html" -%}
+
+  </body>
+</html>
+

examples/templates/app/detail_base.html

@@ -0,0 +1,115 @@
+{# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2016 CERN.
+#
+# Invenio is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+-#}
+
+{%- macro record_content(data) %}
+  {% for key, value in data.items() recursive %}
+    <li class="list-group-item">
+    {% if value is mapping %}
+        <strong>{{ key }}:</strong>
+        <ul class="list-group">{{ loop(value.items()) }}</ul>
+    {% elif value is iterable and value is not string %}
+        <strong>{{ key }}:</strong>
+        <ol>
+        {% for item in value %}
+          <li>
+          {% if item is mapping %}
+            <ul class="list-group">
+              {{ record_content(item) }}
+            </ul>
+          {% else %}
+            {{ item }}
+          {% endif %}
+          </li>
+        {% endfor %}
+        </ol>
+    {% else %}
+      <strong>{{ key }}:</strong> {{ value }}
+    {% endif %}
+    </li>
+  {% endfor %}
+{%- endmacro %}
+
+{%- block page_body %}
+<div class="container">
+  <div class="extra-info">
+    {%- if 'control_number' in record %}
+      {%- block record_control_number %}
+      <b>{{ _('control number:') }}</b>
+      {{ record['control_number'] }}
+      {%- endblock %}
+    {% endif %}
+    <br>
+    {%- if 'publication_distribution_imprint' in record %}
+      {%- block record_publication_date %}
+        <b>{{ _('Publication date:') }}</b>
+        {{ record['publication_distribution_imprint'][0]['date_of_publication_distribution'][0] }}
+      {%- endblock %}
+    {% endif %}
+    <br>
+    {%- if 'other_standard_identifier' in record %}
+      {%- block record_other_standard_identifier %}
+        <b>{{ _('OAI Id:') }}</b>
+        {{ record['other_standard_identifier'][0]['standard_number_or_code'] }}
+      {%- endblock %}
+    {% endif %}
+  </div>
+  <div class="body">
+    {%- if 'title_statement' in record %}
+      {%- block record_title %}
+        <h2>{{ record['title_statement']['title'] }} <br>
+        </h2>
+      {%- endblock %}
+    {% endif %}
+    {%- if 'main_entry_personal_name' in record %}
+      {%- block record_author %}
+        <p><i>
+          {{ record['main_entry_personal_name']['personal_name'] }}
+          {%- if 'added_entry_personal_name' in record %}
+            {{ record['added_entry_personal_name']
+                | map(attribute='personal_name') | join(', ') }}
+          {% endif %}
+        </i></p>
+      {%- endblock %}
+    {% endif %}
+    {%- if 'summary' in record %}
+      {%- block record_summary %}
+        <p>{{ record['summary'][0]['summary'] }}</p>
+      {%- endblock %}
+    {%- endif %}
+  </div>
+</div>
+<div class="raw">
+  {%- block record_raw %}
+  {% if record %}
+    <div class="panel panel-default">
+      <p>Record:</p>
+      <ul class="list-group">
+        {{ record_content(record) }}
+      </ul>
+    </div>
+  {% endif %}
+  {%- endblock %}
+</div>
+{%- endblock %}

requirements-devel.txt

@@ -27,3 +27,4 @@
 #
 #     -e git+git://github.com/mitsuhiko/werkzeug.git#egg=Werkzeug
 #     -e git+git://github.com/mitsuhiko/jinja2.git#egg=Jinja2
+-e git+git://github.com/inveniosoftware/dojson.git#egg=dojson

setup.cfg

@@ -23,14 +23,14 @@
 # as an Intergovernmental Organization or submit itself to any jurisdiction.
 
 
-[aliases]
-test=pytest
-
 [build_sphinx]
 source-dir = docs/
 build-dir = docs/_build
 all_files = 1
 
+[aliases]
+test=pytest
+
 [bdist_wheel]
 universal = 1