collections: hierarchize records

* Adds the possibility to hierarchize collections.
* Displays hierarchized collections in documents aggregations.
* Fixes the link to collections records in document detail view.
* Closes #545.

Co-Authored-by: Sébastien Délèze <sebastien.deleze@rero.ch>

sonar/config.py

@@ -43,7 +43,8 @@
 from sonar.modules.organisations.permissions import OrganisationPermission
 from sonar.modules.permissions import record_permission_factory, \
     wiki_edit_permission
-from sonar.modules.query import and_term_filter, missing_field_filter
+from sonar.modules.query import and_term_filter, collection_filter, \
+    missing_field_filter
 from sonar.modules.users.api import UserRecord, UserSearch
 from sonar.modules.users.permissions import UserPermission
 from sonar.modules.utils import get_current_language
@@ -502,16 +503,41 @@ def _(x):
 RECORDS_REST_FACETS = {
     'documents':
     dict(aggs=dict(
-        sections=dict(terms=dict(field='sections',
-                                     size=DEFAULT_AGGREGATION_SIZE)),
+        sections=dict(
+            terms=dict(field='sections', size=DEFAULT_AGGREGATION_SIZE)),
         organisation=dict(terms=dict(field='organisation.pid',
                                      size=DEFAULT_AGGREGATION_SIZE)),
         language=dict(
             terms=dict(field='language.value', size=DEFAULT_AGGREGATION_SIZE)),
         subject=dict(
             terms=dict(field='facet_subjects', size=DEFAULT_AGGREGATION_SIZE)),
-        collection=dict(terms=dict(field='collections.pid',
-                                            size=DEFAULT_AGGREGATION_SIZE)),
+        collection={
+            'nested': {
+                'path': 'collections'
+            },
+            'aggs': {
+                'collection': {
+                    'terms': {
+                        'field': 'collections.collection0.keyword'
+                    },
+                    'aggs': {
+                        'collection1': {
+                            'terms': {
+                                'field': 'collections.collection1.keyword'
+                            },
+                            'aggs': {
+                                'collection2': {
+                                    'terms': {
+                                        'field':
+                                        'collections.collection2.keyword'
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        },
         document_type=dict(
             terms=dict(field='documentType', size=DEFAULT_AGGREGATION_SIZE)),
         controlled_affiliation=dict(
@@ -540,7 +566,11 @@ def _(x):
              'subject':
              and_term_filter('facet_subjects'),
              'collection':
-             and_term_filter('collections.pid'),
+             collection_filter('collections.collection0.keyword'),
+             'collection1':
+             collection_filter('collections.collection1.keyword'),
+             'collection2':
+             collection_filter('collections.collection2.keyword'),
              'document_type':
              and_term_filter('documentType'),
              'controlled_affiliation':

sonar/ext.py

@@ -29,6 +29,7 @@
 from invenio_indexer.signals import before_record_index
 from werkzeug.datastructures import MIMEAccept
 
+from sonar.modules.collections.receivers import enrich_collection_data
 from sonar.modules.permissions import has_admin_access, has_submitter_access, \
     has_superuser_access
 from sonar.modules.receivers import file_deleted_listener, \
@@ -103,6 +104,9 @@ def init_app(self, app):
         # Add user's full name before record index
         before_record_index.connect(add_full_name, weak=False)
 
+        # Enrich collection's data
+        before_record_index.connect(enrich_collection_data, weak=False)
+
     def init_config(self, app):
         """Initialize configuration."""
         for k in dir(config_sonar):

sonar/modules/collections/jsonschemas/collections/collection-v1.0.0_src.json

@@ -46,7 +46,6 @@
           "language"
         ]
       }
-
     },
     "description": {
       "title": "Descriptions",
@@ -80,6 +79,12 @@
           "value",
           "language"
         ]
+      },
+      "form": {
+        "hide": true,
+        "navigation": {
+          "essential": true
+        }
       }
     },
     "organisation": {
@@ -106,6 +111,33 @@
         }
       }
     },
+    "parent": {
+      "title": "Parent collection",
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "$ref": {
+          "type": "string",
+          "pattern": "^https://sonar.ch/api/collections/.*?$",
+          "form": {
+            "remoteTypeahead": {
+              "type": "collections",
+              "field": "name.value.suggest",
+              "label": "label"
+            }
+          }
+        }
+      },
+      "required": [
+        "$ref"
+      ],
+      "form": {
+        "hide": true,
+        "navigation": {
+          "essential": true
+        }
+      }
+    },
     "_bucket": {
       "title": "Bucket UUID",
       "type": "string",
@@ -169,7 +201,8 @@
   },
   "propertiesOrder": [
     "name",
-    "description"
+    "description",
+    "parent"
   ],
   "required": [
     "name"

sonar/modules/collections/mappings/v7/collections/collection-v1.0.0.json

@@ -54,6 +54,28 @@
           }
         }
       },
+      "parent": {
+        "type": "object",
+        "properties": {
+          "pid": {
+            "type": "keyword"
+          },
+          "name": {
+            "type": "object",
+            "properties": {
+              "value": {
+                "type": "text"
+              },
+              "language": {
+                "type": "keyword"
+              }
+            }
+          }
+        }
+      },
+      "path": {
+        "type": "text"
+      },
       "_created": {
         "type": "date"
       },

sonar/modules/collections/permissions.py

@@ -17,6 +17,8 @@
 
 """Record permissions."""
 
+from elasticsearch_dsl import Q
+
 from sonar.modules.documents.api import DocumentSearch
 from sonar.modules.organisations.api import current_organisation
 from sonar.modules.permissions import RecordPermission as BaseRecordPermission
@@ -92,8 +94,12 @@ def delete(cls, user, record):
         :return: True if action can be done
         :rtype: bool
         """
-        results = DocumentSearch().filter(
-            'term', collections__pid=record['pid']).source(includes=['pid'])
+        results = DocumentSearch().query(
+            Q('nested',
+              path='collections',
+              query=Q('bool', must=Q(
+                  'term',
+                  collections__pid=record['pid'])))).source(includes=['pid'])
 
         # Cannot remove collection associated to a record
         if results.count():

sonar/modules/collections/query.py

@@ -17,7 +17,8 @@
 
 """Query."""
 
-from flask import current_app
+from elasticsearch_dsl import Q
+from flask import current_app, request
 
 from sonar.modules.organisations.api import current_organisation
 from sonar.modules.query import default_search_factory
@@ -36,6 +37,15 @@ def search_factory(self, search):
     if current_app.config.get('SONAR_APP_DISABLE_PERMISSION_CHECKS'):
         return (search, urlkwargs)
 
+    # Cannot suggest a record which is the current collection or the current
+    # collection is one of the parents.
+    if '.suggest' in request.args.get('q',
+                                      '') and request.args.get('currentPid'):
+        search = search.query(
+            Q('bool',
+              must_not=[Q('match',
+                          path='/' + request.args.get('currentPid'))]))
+
     # Records are not filtered for superusers.
     if current_user_record.is_superuser:
         return (search, urlkwargs)

sonar/modules/collections/receivers.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+#
+# Swiss Open Access Repository
+# Copyright (C) 2021 RERO
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Signal receivers for collections."""
+
+
+def enrich_collection_data(sender=None,
+                           record=None,
+                           json=None,
+                           index=None,
+                           **kwargs):
+    """Receive a signal before record is indexed, to enrich the data.
+
+    :param sender: Sender of the signal.
+    :param Record record: Record to index.
+    :param dict json: JSON that will be indexed.
+    :param str index: Name of the index in which record will be sent.
+    """
+    if not index.startswith('collections'):
+        return
+
+    def _build_path(parent, path):
+        """Recursive call to build the path from parent node to leaf.
+
+        :param parent: Parent item.
+        :param path: List of PIDs.
+        """
+        if not parent:
+            return
+
+        path.insert(0, parent['pid'])
+        _build_path(parent.get('parent'), path)
+
+    # Make the pass
+    path = [json['pid']]
+    _build_path(json.get('parent'), path)
+    path.insert(0, '')
+    json['path'] = '/'.join(path)

sonar/modules/collections/schemas.py

@@ -41,6 +41,7 @@ class RecordMetadataSchema(StrictKeysMixin):
     name = fields.List(fields.Dict(), required=True)
     description = fields.List(fields.Dict())
     organisation = fields.Dict()
+    parent = fields.Dict()
     permissions = fields.Dict(dump_only=True)
     label = fields.Method('get_label')
     # When loading, if $schema is not provided, it's retrieved by

sonar/modules/collections/templates/collections/index.html

@@ -18,7 +18,7 @@
 {%- extends config.RECORDS_UI_BASE_TEMPLATE %}
 
 {%- block body %}
-<h1>{{ _('Collections') }}</h1>
+<h1 class="mb-4">{{ _('Collections') }}</h1>
 {% if records | length %}
 <ul class="list-group list-group-flush">
   {% for record in records %}

sonar/modules/collections/templates/collections/item.html

@@ -30,9 +30,6 @@ <h3>{{ name }}</h3>
       {{ collection.description | language_value | markdown_filter | safe }}
     </div>
     {% endif %}
-    <p>
-
-    </p>
     <div class="row">
       <div class="col-sm-6 pt-2">
         <a href="{{ url_for('invenio_records_ui.coll', pid_value=collection.pid, view=view_code) }}">

sonar/modules/collections/views.py

@@ -17,10 +17,12 @@
 
 """Collections views."""
 
+from elasticsearch_dsl import Q
 from flask import Blueprint, abort, current_app, redirect, render_template, \
     url_for
 
 from sonar.modules.collections.api import RecordSearch
+from sonar.modules.documents.api import DocumentSearch
 
 blueprint = Blueprint('collections',
                       __name__,
@@ -43,7 +45,17 @@ def index(**kwargs):
     records = RecordSearch().filter('term',
                                     organisation__pid=kwargs['view']).scan()
 
-    return render_template('collections/index.html', records=list(records))
+    def filter_result(item):
+        """Keep only collections that have documents attached."""
+        documents = DocumentSearch().query(
+            Q('nested',
+              path='collections',
+              query=Q('bool', must=Q('term', collections__pid=item['pid']))))
+
+        return documents.count()
+
+    return render_template('collections/index.html',
+                           records=list(filter(filter_result, records)))
 
 
 def detail(pid, record, **kwargs):

sonar/modules/documents/mappings/v7/documents/document-v1.0.0.json

@@ -313,7 +313,7 @@
         }
       },
       "collections": {
-        "type": "object",
+        "type": "nested",
         "properties": {
           "pid": {
             "type": "keyword"

sonar/modules/documents/query.py

@@ -87,8 +87,14 @@ def search_factory(self, search, query_parser=None):
 
         # Filter collection
         if request.args.get('collection_view'):
-            search = search.filter(
-                'term', collections__pid=request.args['collection_view'])
+            search = search.query(
+                Q('nested',
+                  path='collections',
+                  query=Q(
+                      'bool',
+                      must=Q(
+                          'term',
+                          collections__pid=request.args['collection_view']))))
     # Admin
     else:
         # Filters records by user's organisation

sonar/modules/documents/receivers.py

@@ -109,6 +109,27 @@ def enrich_document_data(sender=None,
     # Check if record is open access.
     json['isOpenAccess'] = record.is_open_access()
 
+    # Adds collections for building aggregations.
+    def _get_collections_pids(collection):
+        """Builds the list of collections PIDs from parent to leaf.
+
+        :param dict collection: Collection dictionary.
+        :returns: List of PIDs.
+        :rtype: list
+        """
+        pids = [collection['pid']]
+        if collection.get('parent'):
+            pids = _get_collections_pids(collection['parent']) + pids
+
+        return pids
+
+    for collection in json.get('collections', []):
+        pids = _get_collections_pids(collection)
+        i = 0
+        while i < len(pids):
+            collection[f'collection{i}'] = pids[i]
+            i += 1
+
     # No files are present in record
     if not record.files:
         return