From 7b2f49b76a02197f11e2bcf50fa9861613d489cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Se=CC=81bastien=20De=CC=81le=CC=80ze?= Date: Mon, 31 May 2021 16:39:15 +0200 Subject: [PATCH] collections: hierarchize records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adds the possibility to hierarchize collections. * Displays hierarchized collections in documents aggregations. * Fixes the link to collections records in document detail view. * Closes #545. Co-Authored-by: Sébastien Délèze --- sonar/config.py | 42 ++++++++++++--- sonar/ext.py | 4 ++ .../collections/collection-v1.0.0_src.json | 37 ++++++++++++- .../v7/collections/collection-v1.0.0.json | 22 ++++++++ sonar/modules/collections/permissions.py | 10 +++- sonar/modules/collections/query.py | 12 ++++- sonar/modules/collections/receivers.py | 52 +++++++++++++++++++ sonar/modules/collections/schemas.py | 1 + .../templates/collections/index.html | 2 +- .../templates/collections/item.html | 3 -- sonar/modules/collections/views.py | 14 ++++- .../v7/documents/document-v1.0.0.json | 2 +- sonar/modules/documents/query.py | 10 +++- sonar/modules/documents/receivers.py | 21 ++++++++ .../modules/documents/serializers/__init__.py | 30 ++++++++--- .../documents/templates/documents/record.html | 2 +- sonar/modules/query.py | 14 +++++ .../test_collections_documents_facets.py | 30 ++++++++++- .../test_collections_suggestions.py | 41 +++++++++++++++ tests/conftest.py | 18 +++++++ .../ui/collections/test_collections_views.py | 22 ++++++-- 21 files changed, 357 insertions(+), 32 deletions(-) create mode 100644 sonar/modules/collections/receivers.py create mode 100644 tests/api/collections/test_collections_suggestions.py diff --git a/sonar/config.py b/sonar/config.py index ef9691a19..5dd503cb7 100644 --- a/sonar/config.py +++ b/sonar/config.py @@ -43,7 +43,8 @@ from sonar.modules.organisations.permissions import OrganisationPermission from sonar.modules.permissions import record_permission_factory, \ wiki_edit_permission -from sonar.modules.query import and_term_filter, missing_field_filter +from sonar.modules.query import and_term_filter, collection_filter, \ + missing_field_filter from sonar.modules.users.api import UserRecord, UserSearch from sonar.modules.users.permissions import UserPermission from sonar.modules.utils import get_current_language @@ -502,16 +503,41 @@ def _(x): RECORDS_REST_FACETS = { 'documents': dict(aggs=dict( - sections=dict(terms=dict(field='sections', - size=DEFAULT_AGGREGATION_SIZE)), + sections=dict( + terms=dict(field='sections', size=DEFAULT_AGGREGATION_SIZE)), organisation=dict(terms=dict(field='organisation.pid', size=DEFAULT_AGGREGATION_SIZE)), language=dict( terms=dict(field='language.value', size=DEFAULT_AGGREGATION_SIZE)), subject=dict( terms=dict(field='facet_subjects', size=DEFAULT_AGGREGATION_SIZE)), - collection=dict(terms=dict(field='collections.pid', - size=DEFAULT_AGGREGATION_SIZE)), + collection={ + 'nested': { + 'path': 'collections' + }, + 'aggs': { + 'collection': { + 'terms': { + 'field': 'collections.collection0.keyword' + }, + 'aggs': { + 'collection1': { + 'terms': { + 'field': 'collections.collection1.keyword' + }, + 'aggs': { + 'collection2': { + 'terms': { + 'field': + 'collections.collection2.keyword' + } + } + } + } + } + } + } + }, document_type=dict( terms=dict(field='documentType', size=DEFAULT_AGGREGATION_SIZE)), controlled_affiliation=dict( @@ -540,7 +566,11 @@ def _(x): 'subject': and_term_filter('facet_subjects'), 'collection': - and_term_filter('collections.pid'), + collection_filter('collections.collection0.keyword'), + 'collection1': + collection_filter('collections.collection1.keyword'), + 'collection2': + collection_filter('collections.collection2.keyword'), 'document_type': and_term_filter('documentType'), 'controlled_affiliation': diff --git a/sonar/ext.py b/sonar/ext.py index d2a840965..66d7179a1 100644 --- a/sonar/ext.py +++ b/sonar/ext.py @@ -29,6 +29,7 @@ from invenio_indexer.signals import before_record_index from werkzeug.datastructures import MIMEAccept +from sonar.modules.collections.receivers import enrich_collection_data from sonar.modules.permissions import has_admin_access, has_submitter_access, \ has_superuser_access from sonar.modules.receivers import file_deleted_listener, \ @@ -103,6 +104,9 @@ def init_app(self, app): # Add user's full name before record index before_record_index.connect(add_full_name, weak=False) + # Enrich collection's data + before_record_index.connect(enrich_collection_data, weak=False) + def init_config(self, app): """Initialize configuration.""" for k in dir(config_sonar): diff --git a/sonar/modules/collections/jsonschemas/collections/collection-v1.0.0_src.json b/sonar/modules/collections/jsonschemas/collections/collection-v1.0.0_src.json index 690655016..4bdad8483 100644 --- a/sonar/modules/collections/jsonschemas/collections/collection-v1.0.0_src.json +++ b/sonar/modules/collections/jsonschemas/collections/collection-v1.0.0_src.json @@ -46,7 +46,6 @@ "language" ] } - }, "description": { "title": "Descriptions", @@ -80,6 +79,12 @@ "value", "language" ] + }, + "form": { + "hide": true, + "navigation": { + "essential": true + } } }, "organisation": { @@ -106,6 +111,33 @@ } } }, + "parent": { + "title": "Parent collection", + "type": "object", + "additionalProperties": false, + "properties": { + "$ref": { + "type": "string", + "pattern": "^https://sonar.ch/api/collections/.*?$", + "form": { + "remoteTypeahead": { + "type": "collections", + "field": "name.value.suggest", + "label": "label" + } + } + } + }, + "required": [ + "$ref" + ], + "form": { + "hide": true, + "navigation": { + "essential": true + } + } + }, "_bucket": { "title": "Bucket UUID", "type": "string", @@ -169,7 +201,8 @@ }, "propertiesOrder": [ "name", - "description" + "description", + "parent" ], "required": [ "name" diff --git a/sonar/modules/collections/mappings/v7/collections/collection-v1.0.0.json b/sonar/modules/collections/mappings/v7/collections/collection-v1.0.0.json index b55961db2..4e6d306a6 100644 --- a/sonar/modules/collections/mappings/v7/collections/collection-v1.0.0.json +++ b/sonar/modules/collections/mappings/v7/collections/collection-v1.0.0.json @@ -54,6 +54,28 @@ } } }, + "parent": { + "type": "object", + "properties": { + "pid": { + "type": "keyword" + }, + "name": { + "type": "object", + "properties": { + "value": { + "type": "text" + }, + "language": { + "type": "keyword" + } + } + } + } + }, + "path": { + "type": "text" + }, "_created": { "type": "date" }, diff --git a/sonar/modules/collections/permissions.py b/sonar/modules/collections/permissions.py index 79f205fa1..89c322393 100644 --- a/sonar/modules/collections/permissions.py +++ b/sonar/modules/collections/permissions.py @@ -17,6 +17,8 @@ """Record permissions.""" +from elasticsearch_dsl import Q + from sonar.modules.documents.api import DocumentSearch from sonar.modules.organisations.api import current_organisation from sonar.modules.permissions import RecordPermission as BaseRecordPermission @@ -92,8 +94,12 @@ def delete(cls, user, record): :return: True if action can be done :rtype: bool """ - results = DocumentSearch().filter( - 'term', collections__pid=record['pid']).source(includes=['pid']) + results = DocumentSearch().query( + Q('nested', + path='collections', + query=Q('bool', must=Q( + 'term', + collections__pid=record['pid'])))).source(includes=['pid']) # Cannot remove collection associated to a record if results.count(): diff --git a/sonar/modules/collections/query.py b/sonar/modules/collections/query.py index 1b915fe5b..a3d68db35 100644 --- a/sonar/modules/collections/query.py +++ b/sonar/modules/collections/query.py @@ -17,7 +17,8 @@ """Query.""" -from flask import current_app +from elasticsearch_dsl import Q +from flask import current_app, request from sonar.modules.organisations.api import current_organisation from sonar.modules.query import default_search_factory @@ -36,6 +37,15 @@ def search_factory(self, search): if current_app.config.get('SONAR_APP_DISABLE_PERMISSION_CHECKS'): return (search, urlkwargs) + # Cannot suggest a record which is the current collection or the current + # collection is one of the parents. + if '.suggest' in request.args.get('q', + '') and request.args.get('currentPid'): + search = search.query( + Q('bool', + must_not=[Q('match', + path='/' + request.args.get('currentPid'))])) + # Records are not filtered for superusers. if current_user_record.is_superuser: return (search, urlkwargs) diff --git a/sonar/modules/collections/receivers.py b/sonar/modules/collections/receivers.py new file mode 100644 index 000000000..ba167742d --- /dev/null +++ b/sonar/modules/collections/receivers.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# +# Swiss Open Access Repository +# Copyright (C) 2021 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Signal receivers for collections.""" + + +def enrich_collection_data(sender=None, + record=None, + json=None, + index=None, + **kwargs): + """Receive a signal before record is indexed, to enrich the data. + + :param sender: Sender of the signal. + :param Record record: Record to index. + :param dict json: JSON that will be indexed. + :param str index: Name of the index in which record will be sent. + """ + if not index.startswith('collections'): + return + + def _build_path(parent, path): + """Recursive call to build the path from parent node to leaf. + + :param parent: Parent item. + :param path: List of PIDs. + """ + if not parent: + return + + path.insert(0, parent['pid']) + _build_path(parent.get('parent'), path) + + # Make the pass + path = [json['pid']] + _build_path(json.get('parent'), path) + path.insert(0, '') + json['path'] = '/'.join(path) diff --git a/sonar/modules/collections/schemas.py b/sonar/modules/collections/schemas.py index cbab54fbb..20a0257fb 100644 --- a/sonar/modules/collections/schemas.py +++ b/sonar/modules/collections/schemas.py @@ -41,6 +41,7 @@ class RecordMetadataSchema(StrictKeysMixin): name = fields.List(fields.Dict(), required=True) description = fields.List(fields.Dict()) organisation = fields.Dict() + parent = fields.Dict() permissions = fields.Dict(dump_only=True) label = fields.Method('get_label') # When loading, if $schema is not provided, it's retrieved by diff --git a/sonar/modules/collections/templates/collections/index.html b/sonar/modules/collections/templates/collections/index.html index 5843e9894..f9ffe7de0 100644 --- a/sonar/modules/collections/templates/collections/index.html +++ b/sonar/modules/collections/templates/collections/index.html @@ -18,7 +18,7 @@ {%- extends config.RECORDS_UI_BASE_TEMPLATE %} {%- block body %} -

{{ _('Collections') }}

+

{{ _('Collections') }}

{% if records | length %}