Merge pull request #62 from scossu/development

Development

VERSION

@@ -1 +1 @@
-1.0.0a13
+1.0.0a14

docs/discovery.rst

@@ -66,7 +66,23 @@ or a native Python method if applicable.
 Term Search
 -----------
 
-This feature has not yet been implemented. It is meant to provide a discovery
-tool based on simple term match, and possibly comparison. It should be more
-efficient and predictable than SPARQL.
+.. figure:: assets/lsup_term_search.png
+   :alt: LAKEsuperior Term Search Window
+
+   LAKEsuperior Term Search Window
+
+This feature provides a discovery tool focused on resource subjects and based
+on individual term match and comparison. It tends to be more manageable than
+SPARQL but also uses some SPARQL syntax for the terms.
+
+Multiple search conditions can be entered and processed with AND or OR logic.
+
+The obtained results are resource URIs relative to the endpoint.
+
+Please consult the search page itself for detailed instructions on how to enter
+query terms.
+
+The term search is also available via REST API. E.g.::
+
+    curl -i -XPOST http://localhost:8000/query/term_search -d '{"terms": [{"pred": "rdf:type", "op": "_id", "val": "ldp:Container"}], "logic": "and"}' -H'Content-Type:application/json'
 

docs/usage.rst

@@ -152,10 +152,22 @@ Create an LDP-NR (non-RDF source)::
     ...     uid, stream=BytesIO(data), mimetype='text/plain')
     '_create_'
 
+Create or replace providing a serialized RDF byte stream::
+
+    >>> uid = '/rsrc_from_rdf'
+    >>> rdf = b'<#a1> a <http://ex.org/type#B> .'
+    >>> rsrc_api.create_or_replace(uid, rdf_data=rdf, rdf_fmt='turtle')
+
+Relative URIs such as ``<#a1>`` will be resolved relative to the resource URI.
+
 Create under a known parent, providing a slug (POST style)::
 
     >>> rsrc_api.create('/rsrc_from_stream', 'res1')
 
+This will create ``/rsrc_from_stream/res1`` if not existing; otherwise the
+resource URI will have a random UUID4 instead of ``res1``.
+
+To use a random UUID by default, use ``None`` for the second argument.
 
 Retrieve Resources
 ~~~~~~~~~~~~~~~~~~

lakesuperior/api/query.py

@@ -2,6 +2,8 @@
 
 from io import BytesIO
 
+from rdflib import URIRef
+
 from lakesuperior import env
 from lakesuperior.dictionaries.namespaces import ns_collection as nsc
 from lakesuperior.dictionaries.namespaces import ns_mgr as nsm
@@ -12,6 +14,103 @@
 rdfly = env.app_globals.rdfly
 rdf_store = env.app_globals.rdf_store
 
+operands = ('_id', '=', '!=', '<', '>', '<=', '>=')
+"""
+Available term comparators for term query.
+
+The ``_uri`` term is used to match URIRef terms, all other comparators are
+used against literals.
+"""
+
+
+def triple_match(s=None, p=None, o=None, return_full=False):
+    """
+    Query store by matching triple patterns.
+
+    Any of the ``s``, ``p`` or ``o`` terms can be None to represent a wildcard.
+
+    This method is for triple matching only; it does not allow to query, nor
+    exposes to the caller, any context.
+
+    :param rdflib.term.Identifier s: Subject term.
+    :param rdflib.term.Identifier p: Predicate term.
+    :param rdflib.term.Identifier o: Object term.
+    :param bool return_full: if ``False`` (the default), the returned values
+        in the set are the URIs of the resources found. If True, the full set
+        of matching triples is returned.
+
+    :rtype: set(tuple(rdflib.term.Identifier){3}) or set(rdflib.URIRef)
+    :return: Matching resource URIs if ``return_full`` is false, or
+        matching triples otherwise.
+    """
+    with TxnManager(rdf_store) as txn:
+        matches = rdf_store.triples((s, p, o), None)
+        # Strip contexts and de-duplicate.
+        qres = (
+            {match[0] for match in matches} if return_full
+            else {match[0][0] for match in matches})
+
+    return qres
+
+
+def term_query(terms, or_logic=False):
+    """
+    Query resources by predicates, comparators and values.
+
+    Comparators can be against literal or URIRef objects. For a list of
+    comparators and their meanings, see the documentation and source for
+    :py:data:`~lakesuperior.api.query.operands`.
+
+    :param list(tuple{3}) terms: List of 3-tuples containing:
+
+        - Predicate URI (rdflib.URIRef)
+        - Comparator value (str)
+        - Value to compare to (rdflib.URIRef or rdflib.Literal or str)
+
+    :param bool or_logic: Whether to concatenate multiple query terms with OR
+        logic (uses SPARQL ``UNION`` statements). The default is False (i.e.
+        terms are concatenated as standard SPARQL statements).
+    """
+    qry_term_ls = []
+    for i, term in enumerate(terms):
+        if term['op'] not in operands:
+            raise ValueError('Not a valid operand: {}'.format(term['op']))
+
+        if term['op'] == '_id':
+            qry_term = '?s {} {} .'.format(term['pred'], term['val'])
+        else:
+            oname = '?o_{}'.format(i)
+            qry_term = '?s {0} {1}\nFILTER (str({1}) {2} "{3}") .'.format(
+                    term['pred'], oname, term['op'], term['val'])
+
+        qry_term_ls.append(qry_term)
+
+    if or_logic:
+        qry_terms = '{\n' + '\n} UNION {\n'.join(qry_term_ls) + '\n}'
+    else:
+        qry_terms = '\n'.join(qry_term_ls)
+    qry_str = '''
+    SELECT ?s WHERE {{
+      {}
+    }}
+    '''.format(qry_terms)
+    logger.debug('Query: {}'.format(qry_str))
+
+    with TxnManager(rdf_store) as txn:
+        qres = rdfly.raw_query(qry_str)
+        return {row[0] for row in qres}
+
+
+def fulltext_lookup(pattern):
+    """
+    Look up one term by partial match.
+
+    *TODO: reserved for future use. A `Whoosh
+    <https://whoosh.readthedocs.io/>`__ or similar full-text index is
+    necessary for this.*
+    """
+    pass
+
 
 def sparql_query(qry_str, fmt):
     """

lakesuperior/dictionaries/namespaces.py

@@ -34,7 +34,8 @@
 }
 
 ns_collection = core_namespaces.copy()
-ns_collection.update(config['namespaces'])
+custom_ns = {pfx: Namespace(ns) for pfx, ns in config['namespaces'].items()}
+ns_collection.update(custom_ns)
 
 ns_mgr = NamespaceManager(Graph())
 ns_pfx_sparql = {}

lakesuperior/endpoints/ldp.py

@@ -228,16 +228,17 @@ def post_resource(parent_uid):
     if LdpFactory.is_rdf_parsable(mimetype):
         # If the content is RDF, localize in-repo URIs.
         global_rdf = stream.read()
-        local_rdf = g.tbox.localize_payload(global_rdf)
-        stream = BytesIO(local_rdf)
-        is_rdf = True
+        rdf_data = g.tbox.localize_payload(global_rdf)
+        rdf_fmt = mimetype
+        stream = mimetype = None
     else:
-        is_rdf = False
+        rdf_data = rdf_fmt = None
 
     try:
         uid = rsrc_api.create(
-                parent_uid, slug, stream=stream, mimetype=mimetype,
-                handling=handling, disposition=disposition)
+            parent_uid, slug, stream=stream, mimetype=mimetype,
+            rdf_data=rdf_data, rdf_fmt=rdf_fmt, handling=handling,
+            disposition=disposition)
     except ResourceNotExistsError as e:
         return str(e), 404
     except InvalidResourceError as e:
@@ -279,16 +280,17 @@ def put_resource(uid):
     if LdpFactory.is_rdf_parsable(mimetype):
         # If the content is RDF, localize in-repo URIs.
         global_rdf = stream.read()
-        local_rdf = g.tbox.localize_payload(global_rdf)
-        graph = Graph().parse(
-                data=local_rdf, format=mimetype, publicID=nsc['fcres'][uid])
+        rdf_data = g.tbox.localize_payload(global_rdf)
+        rdf_fmt = mimetype
         stream = mimetype = None
     else:
-        graph = None
+        rdf_data = rdf_fmt = None
 
     try:
-        evt = rsrc_api.create_or_replace(uid, stream=stream, mimetype=mimetype,
-                graph=graph, handling=handling, disposition=disposition)
+        evt = rsrc_api.create_or_replace(
+            uid, stream=stream, mimetype=mimetype,
+            rdf_data=rdf_data, rdf_fmt=rdf_fmt, handling=handling,
+            disposition=disposition)
     except (InvalidResourceError, ResourceExistsError) as e:
         return str(e), 409
     except (ServerManagedTermError, SingleSubjectError) as e:
@@ -302,7 +304,7 @@ def put_resource(uid):
     if evt == RES_CREATED:
         rsp_code = 201
         rsp_headers['Location'] = rsp_body = uri
-        if mimetype and not graph:
+        if mimetype and not rdf_data:
             rsp_headers['Link'] = (
                     '<{0}/fcr:metadata>; rel="describedby"'.format(uri))
     else:

lakesuperior/endpoints/query.py

@@ -1,11 +1,16 @@
 import logging
 
-from flask import Blueprint, current_app, request, render_template, send_file
+from flask import (
+        Blueprint, current_app, jsonify, request, make_response,
+        render_template, send_file)
+from rdflib import URIRef
 from rdflib.plugin import PluginException
 
 from lakesuperior import env
-from lakesuperior.dictionaries.namespaces import ns_mgr as nsm
 from lakesuperior.api import query as query_api
+from lakesuperior.dictionaries.namespaces import ns_collection as nsc
+from lakesuperior.dictionaries.namespaces import ns_mgr as nsm
+from lakesuperior.toolbox import Toolbox
 
 # Query endpoint. raw SPARQL queries exposing the underlying layout can be made
 # available. Also convenience methods that allow simple lookups based on simple
@@ -18,24 +23,37 @@
 query = Blueprint('query', __name__)
 
 
-@query.route('/term_search', methods=['GET'])
+@query.route('/term_search', methods=['GET', 'POST'])
 def term_search():
     """
     Search by entering a search term and optional property and comparison term.
     """
-    valid_operands = (
-        ('=', 'Equals'),
-        ('>', 'Greater Than'),
-        ('<', 'Less Than'),
-        ('<>', 'Not Equal'),
-        ('a', 'RDF Type'),
+    operands = (
+        ('_id', 'Matches Term'),
+        ('=', 'Is Equal To'),
+        ('!=', 'Is Not Equal To'),
+        ('<', 'Is Less Than'),
+        ('>', 'Is Greater Than'),
+        ('<=', 'Is Less Than Or Equal To'),
+        ('>=', 'Is Greater Than Or Equal To'),
     )
+    qres = term_list = []
 
-    term = request.args.get('term')
-    prop = request.args.get('prop', default=1)
-    cmp = request.args.get('cmp', default='=')
+    if request.method == 'POST':
+        terms = request.json.get('terms', {})
+        or_logic = request.json.get('logic', 'and') == 'or'
+        logger.info('Form: {}'.format(request.json))
+        logger.info('Terms: {}'.format(terms))
+        logger.info('Logic: {}'.format(or_logic))
+        qres = query_api.term_query(terms, or_logic)
 
-    return render_template('term_search.html')
+        rsp = [
+            uri.replace(nsc['fcres'], request.host_url.rstrip('/') + '/ldp')
+            for uri in qres]
+        return jsonify(rsp), 200
+    else:
+        return render_template(
+            'term_search.html', operands=operands, qres=qres, nsm=nsm)
 
 
 @query.route('/sparql', methods=['GET', 'POST'])

lakesuperior/endpoints/templates/base.html

@@ -14,49 +14,16 @@
     {% endblock %}
     </head>
     <body>
-    {% block navbar%}
-        <nav class="navbar navbar-default">
-          <div class="container-fluid">
-            <!-- Brand and toggle get grouped for better mobile display -->
-            <div class="navbar-header">
-              <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">Toggle navigation</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-              </button>
-              <a class="navbar-brand" href="/">LAKEsuperior</a>
-            </div>
-            <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-              <ul class="nav navbar-nav">
-                <li><a href="/ldp">Browse Resources<span class="sr-only">(current)</span></a></li>
-                <li class="dropdown">
-                  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Query<span class="caret"></span></a>
-                  <ul class="dropdown-menu">
-                    <li><a href="/query/term_search">Term Search</a></li>
-                    <li><a href="/query/sparql">SPARQL Query</a></li>
-                  </ul>
-                </li>
-                <li class="dropdown">
-                  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Administration<span class="caret"></span></a>
-                  <ul class="dropdown-menu">
-                    <li><a href="/admin/stats">Statistics</a></li>
-                    <li><a href="/admin/tools">Tools</a></li>
-                  </ul>
-                </li>
-              </ul>
-            </div><!-- /.navbar-collapse -->
-          </div><!-- /.container-fluid -->
-        </nav>
-    {% endblock %}
-        <main class="container">
+        {% include 'navbar.html' %}
+        <div class="container">
             {% block breadcrumbs %}{% endblock %}
             <h1>{{ self.title() }}</h1>
             {% block content %}{% endblock %}
-        </main>
+        </div>
         <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
         <script src="{{url_for('ldp.static', filename='assets/js/jquery-3.2.1.min.js')}}"></script>
         <!-- Include all compiled plugins (below), or include individual files as needed -->
-        <script src="{{url_for('ldp.static', filename='assets/js/bootstrap.min.js')}}"></script>
+        <script src="{{url_for('ldp.static', filename='assets/js/bootstrap.bundle.min.js')}}"></script>
+        {% block tail_js %}{% endblock %}
     </body>
 </html>

lakesuperior/endpoints/templates/namespaces.html

@@ -1,7 +1,9 @@
 <h2>Namespaces</h2>
- <button class="btn btn-primary" type="button" data-toggle="collapse" data-target="#nslist" aria-expanded="false" aria-controls="nsList">
+<div class="my-sm-3">
+  <button class="btn btn-primary" type="button" data-toggle="collapse" data-target="#nslist" aria-expanded="false" aria-controls="nsList">
     Expand/Collapse
-</button>
+  </button>
+</div>
 <div class="collapse" id="nslist">
     <div class="card card-body">
         <table class="table table-striped">