files: uses references for library and document

* Replaces owners and links to library and document $ref in the record file. * Reverts uwsgi files modified by mistake. * Fixes spelling in some function names. Co-Authored-by: Johnny Mariéthoz <Johnny.Mariethoz@rero.ch>
rero · May 15, 2024 · 6ff907d · 6ff907d
1 parent bf97cc0
commit 6ff907d
Show file tree

Hide file tree

Showing 24 changed files with 180 additions and 106 deletions.
diff --git a/docker/uwsgi/uwsgi_rest.ini b/docker/uwsgi/uwsgi_rest.ini
@@ -16,7 +16,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 [uwsgi]
-socket = localhost:5001
+socket = 0.0.0.0:5001
 module = invenio_app.wsgi_rest:application
 master = true
 die-on-term = true

diff --git a/docker/uwsgi/uwsgi_ui.ini b/docker/uwsgi/uwsgi_ui.ini
@@ -16,7 +16,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 [uwsgi]
-socket = localhost:5000
+socket = 0.0.0.0:5000
 module = invenio_app.wsgi_ui:application
 master = true
 die-on-term = true

diff --git a/rero_ils/config.py b/rero_ils/config.py
@@ -1249,7 +1249,7 @@ def _(x):
         item_route='/local_entities/<pid(locent, record_class="rero_ils.modules.entities.local_entities.api:LocalEntity"):pid_value>',
         default_media_type='application/json',
         max_result_window=MAX_RESULT_WINDOW,
-        search_factory_imp='rero_ils.query:search_factory',
+        search_factory_imp='rero_ils.query:remote_entity_view_search_factory',
         list_permission_factory_imp=lambda record: LocalEntityPermissionPolicy('search', record=record),
         read_permission_factory_imp=lambda record: LocalEntityPermissionPolicy('read', record=record),
         create_permission_factory_imp=lambda record: LocalEntityPermissionPolicy('create', record=record),
@@ -1274,7 +1274,7 @@ def _(x):
         item_route='/entities/<nooppid(ent, data={}):pid_value>',  # mandatory for invenio-records-rest (only used for permissions)
         default_media_type='application/json',
         max_result_window=MAX_RESULT_WINDOW,
-        search_factory_imp='rero_ils.query:search_factory',
+        search_factory_imp='rero_ils.query:remote_entity_view_search_factory',
         list_permission_factory_imp=allow_all,
         read_permission_factory_imp=deny_all,
         create_permission_factory_imp=deny_all,

diff --git a/rero_ils/modules/commons/dumpers.py b/rero_ils/modules/commons/dumpers.py
@@ -73,4 +73,5 @@ def dump(self, record, data):
         :return a dict with dumped data.
         """
         from copy import deepcopy
-        return deepcopy(_records_state.replace_refs(data))
+        data = deepcopy(_records_state.replace_refs(data))
+        return data
diff --git a/rero_ils/modules/documents/api.py b/rero_ils/modules/documents/api.py
@@ -376,13 +376,10 @@ def get_records_files_query(self, lib_pids=None):
         # TODO: remove this once the issue is solved
         search._params = {}
         search = search.source(['uuid', 'id'])\
-            .filter('term', metadata__links=f'doc_{self.pid}')
+            .filter('term', metadata__document__pid=self.pid)
         # filter by library pids
         if lib_pids:
-            # add lib_ prefix if it is needed
-            if not lib_pids[0].startswith('lib_'):
-                lib_pids = [f'lib_{pid}' for pid in lib_pids]
-            search = search.filter('terms', metadata__owners=lib_pids)
+            search = search.filter('terms', metadata__library__pid=lib_pids)
         return search
 
     def get_records_files(self, lib_pids=None):

diff --git a/rero_ils/modules/documents/dumpers/indexer.py b/rero_ils/modules/documents/dumpers/indexer.py
@@ -22,6 +22,7 @@
 from invenio_records.dumpers import Dumper
 
 from rero_ils.modules.libraries.api import Library
+from rero_ils.modules.utils import extracted_data_from_ref
 
 from ..extensions import TitleExtension
 from ..utils import process_i18n_literal_fields
@@ -236,8 +237,8 @@ def _process_files(self, record, data):
         for record_file in record.get_records_files():
             record_files_information = {}
             collections = record_file.get('metadata', {}).get('collections')
-            owners = record_file.get('metadata', {}).get('owners')
-            library_pid = owners[0].replace('lib_', '')
+            library_pid = extracted_data_from_ref(
+                record_file.get('metadata', {}).get('library'))
             if library_pid:
                 organisation_pid = Library.get_record_by_pid(
                     library_pid).organisation_pid

diff --git a/rero_ils/modules/documents/utils.py b/rero_ils/modules/documents/utils.py
@@ -28,13 +28,13 @@
 from invenio_jsonschemas.proxies import current_jsonschemas
 from werkzeug.local import LocalProxy
 
-from ..utils import get_schema_for_resource, memoized
+from ..utils import get_schema_for_resource, memoize
 from ...utils import get_i18n_supported_languages
 
 _records_state = LocalProxy(lambda: current_app.extensions['invenio-records'])
 
 
-@memoized(timeout=3600)
+@memoize(timeout=3600)
 def get_document_types_from_schema(schema='doc'):
     """Create document type definition from schema."""
     path = current_jsonschemas.url_to_path(get_schema_for_resource(schema))

diff --git a/rero_ils/modules/files/cli.py b/rero_ils/modules/files/cli.py
@@ -32,6 +32,7 @@
 from rero_ils.modules.documents.api import Document
 from rero_ils.modules.documents.dojson.contrib.jsontodc import dublincore
 from rero_ils.modules.libraries.api import Library
+from rero_ils.modules.utils import extracted_data_from_ref, get_ref_for_pid
 
 
 def create_pdf_file(document):
@@ -69,7 +70,8 @@ def create_pdf_record_files(document, metadata, flush=False,
     :param file_name_suffix: str - a suffix to add to the file name.
     """
     # add document link
-    metadata.setdefault("links", [f"doc_{document.pid}"])
+    metadata.setdefault(
+        "document", {'$ref': get_ref_for_pid('doc', document.pid)})
     ext = current_app.extensions["rero-invenio-files"]
     # get services
     record_service = ext.records_service
@@ -78,7 +80,9 @@ def create_pdf_record_files(document, metadata, flush=False,
     content = create_pdf_file(document)
     # create the record file
     try:
-        record = next(document.get_records_files(lib_pids=metadata["owners"]))
+        record = next(
+            document.get_records_files(
+                lib_pids=[extracted_data_from_ref(metadata.get('library'))]))
     except StopIteration:
         item = record_service.create(
             identity=system_identity, data={"metadata": metadata})
@@ -129,13 +133,16 @@ def load_files_for_document(document, metadata, files):
     :param metadata: dict - record metadata.
     :param files: list of str - file paths.
     """
-    metadata.setdefault("links", [f"doc_{document.pid}"])
+    metadata.setdefault(
+        "document", {"$ref": get_ref_for_pid('doc', document.pid)})
     ext = current_app.extensions["rero-invenio-files"]
     # get services
     record_service = ext.records_service
     file_service = ext.records_files_service
     try:
-        record = next(document.get_records_files(lib_pids=metadata["owners"]))
+        record = next(
+            document.get_records_files(
+                lib_pids=[extracted_data_from_ref(metadata.get('library'))]))
     except StopIteration:
         item = record_service.create(
             identity=system_identity, data={"metadata": metadata})
@@ -199,7 +206,8 @@ def create_files(number, collections):
         click.echo(f"Create {number_of_files} files for {pid}")
         lib_pid = choice(lib_pids)
         metadata = dict(
-            collections=[choice(collections)], owners=[f"lib_{lib_pid}"])
+            collections=[choice(collections)],
+            library={'$ref': get_ref_for_pid('lib', lib_pid)})
         create_pdf_record_files(
             document=doc, metadata=metadata, flush=True,
             number_of_files=number_of_files
@@ -220,7 +228,8 @@ def load_files(document_pid, library_pid, files, collections):
     """
     doc = Document.get_record_by_pid(document_pid)
     metadata = dict(
-        links=[f"doc_{document_pid}"], owners=[f"lib_{library_pid}"])
+        document={"$ref": get_ref_for_pid('doc', document_pid)},
+        library={"$ref": get_ref_for_pid('lib', library_pid)})
     if collections:
         metadata["collections"] = collections
     click.secho(f"Loading {len(files)} files...", fg="green")

diff --git a/rero_ils/modules/files/components.py b/rero_ils/modules/files/components.py
@@ -27,6 +27,7 @@
 from rero_ils.modules.operation_logs.extensions import OperationLogFactory
 from rero_ils.modules.operation_logs.logs.api import SpecificOperationLog
 from rero_ils.modules.operation_logs.models import OperationLogOperation
+from rero_ils.modules.utils import extracted_data_from_ref
 
 from .operations import ReindexDoc, ReindexRecordFile
 
@@ -70,14 +71,13 @@ class provider:
 
         rec = Rec()
         rec['pid'] = record.pid.pid_value
-        if owners := record.get('metadata', {}).get('owners'):
-            lib_pid = owners[0].replace('lib_', '')
-            rec.library_pid = lib_pid
+        if library := record.get('metadata', {}).get('library'):
+            rec.library_pid = extracted_data_from_ref(library)
             rec.organisation_pid = Library.get_record_by_pid(
-                lib_pid).organisation_pid
-        if links := record.get('metadata', {}).get('links'):
+                rec.library_pid).organisation_pid
+        if document := record.get('metadata', {}).get('document'):
             rec['document'] = Document.get_record_by_pid(
-                links[0].replace('doc_', ''))
+                extracted_data_from_ref(document))
         OperationLogRecordFactory().create_operation_log(rec, operation)
 
     def create(self, identity, data, record, errors=None, **kwargs):
@@ -141,15 +141,13 @@ class provider:
 
         rec = Rec()
         rec['pid'] = file_key
-        if owners := record.get('metadata', {}).get('owners'):
-            lib_pid = owners[0].replace('lib_', '')
-            rec.library_pid = lib_pid
+        if library := record.get('metadata', {}).get('library'):
+            rec.library_pid = extracted_data_from_ref(library)
             rec.organisation_pid = Library.get_record_by_pid(
-                lib_pid).organisation_pid
-
-        if links := record.get('metadata', {}).get('links'):
+                rec.library_pid).organisation_pid
+        if document := record.get('metadata', {}).get('document'):
             rec['document'] = Document.get_record_by_pid(
-                links[0].replace('doc_', ''))
+                extracted_data_from_ref(document))
         rec['recid'] = record['id']
         OperationLogRecordFactory().create_operation_log(
             record=rec, operation=operation)
@@ -187,7 +185,7 @@ def _register(self, record):
 
         :param record: obj - record instance.
         """
-        doc_pid = record["metadata"]["links"][0].replace("doc_", "")
+        doc_pid = extracted_data_from_ref(record["metadata"]["document"])
         for operation in [ReindexRecordFile(record.id), ReindexDoc(doc_pid)]:
             if operation not in self.uow._operations:
                 self.uow.register(operation)
@@ -243,7 +241,7 @@ def _register(self, record):
 
         :param record: obj - record instance.
         """
-        doc_pid = record["metadata"]["links"][0].replace("doc_", "")
+        doc_pid = extracted_data_from_ref(record["metadata"]["document"])
         for operation in [ReindexDoc(doc_pid)]:
             if operation not in self.uow._operations:
                 self.uow.register(operation)

diff --git a/rero_ils/modules/files/dumpers.py b/rero_ils/modules/files/dumpers.py
@@ -17,6 +17,9 @@
 
 """Files indexer dumpers."""
 
+from copy import deepcopy
+
+from invenio_records.api import _records_state
 from invenio_records.dumpers import SearchDumperExt
 
 
@@ -29,6 +32,7 @@ def dump(self, record, data):
         :param record: The record to dump.
         :param data: The initial dump data passed in by ``record.dumps()``.
         """
+        data.update(deepcopy(_records_state.replace_refs(data)))
         n_main_files = 0
         size = 0
         # inject files informations
@@ -43,8 +47,7 @@ def dump(self, record, data):
                 size += record.files[f].file.size
         data["metadata"]["n_files"] = n_main_files
         data["metadata"]["file_size"] = size
-        lib_pid = data["metadata"]["owners"][0].replace("lib_", "")
+        lib_pid = data["metadata"]["library"]["pid"]
         from rero_ils.modules.libraries.api import Library
         org_pid = Library.get_record_by_pid(lib_pid).organisation_pid
-        data["metadata"]["organisation_pid"] = org_pid
-        data["metadata"]["library_pid"] = lib_pid
+        data["metadata"]["organisation"] = {"pid": org_pid, "type": "doc"}
diff --git a/rero_ils/modules/files/jsonschemas/files/record-v1.0.0.json b/rero_ils/modules/files/jsonschemas/files/record-v1.0.0.json
@@ -13,6 +13,10 @@
     "metadata": {
       "type": "object",
       "additionalProperties": false,
+      "required": [
+        "library",
+        "document"
+      ],
       "properties": {
         "collections": {
           "title": "Collections",
@@ -24,24 +28,26 @@
             "minLength": 1
           }
         },
-        "owners": {
-          "title": "Owners",
-          "type": "array",
-          "minItems": 1,
-          "items": {
-            "title": "Owner",
-            "type": "string",
-            "minLength": 1
+        "document": {
+          "title": "Parent document",
+          "type": "object",
+          "properties": {
+            "$ref": {
+              "title": "Parent document URI",
+              "type": "string",
+              "pattern": "^https://bib.rero.ch/api/documents/.*?$"
+            }
           }
         },
-        "links": {
-          "title": "Links",
-          "type": "array",
-          "minItems": 1,
-          "items": {
-            "title": "Link",
-            "type": "string",
-            "minLength": 1
+        "library": {
+          "title": "Library",
+          "type": "object",
+          "properties": {
+            "$ref": {
+              "title": "Library URI",
+              "type": "string",
+              "pattern": "^https://bib.rero.ch/api/libraries/.*?$"
+            }
           }
         }
       }

diff --git a/rero_ils/modules/files/mappings/v7/files/record-v1.0.0.json b/rero_ils/modules/files/mappings/v7/files/record-v1.0.0.json
@@ -49,13 +49,37 @@
       "metadata": {
         "type": "object",
         "properties": {
-          "owners": {
-            "type": "keyword"
+          "document": {
+            "properties": {
+              "pid": {
+                "type": "keyword"
+              },
+              "type": {
+                "type": "keyword"
+              }
+            }
           },
-          "collections": {
-            "type": "keyword"
+          "library": {
+            "properties": {
+              "pid": {
+                "type": "keyword"
+              },
+              "type": {
+                "type": "keyword"
+              }
+            }
           },
-          "links": {
+          "organisation": {
+            "properties": {
+              "pid": {
+                "type": "keyword"
+              },
+              "type": {
+                "type": "keyword"
+              }
+            }
+          },
+          "collections": {
             "type": "keyword"
           },
           "n_files": {
@@ -71,4 +95,4 @@
       }
     }
   }
-}
+}
diff --git a/rero_ils/modules/files/permissions.py b/rero_ils/modules/files/permissions.py
@@ -22,6 +22,7 @@
 
 from rero_ils.modules.permissions import AllowedByAction, \
     AllowedByActionRestrictByManageableLibrary, RecordPermissionPolicy
+from rero_ils.modules.utils import extracted_data_from_ref
 
 # Actions to control Record Files policies for CRUD operations
 search_action = action_factory("file-search")
@@ -39,7 +40,7 @@ def get_library_pid(record):
     @returns: the library pid value.
     @rtype: string.
     """
-    return record.get("metadata", {}).get("owners", [])[0].replace("lib_", "")
+    return extracted_data_from_ref(record.get("metadata", {}).get("library"))
 
 
 class FilePermissionPolicy(RecordPermissionPolicy):
@@ -64,7 +65,7 @@ class FilePermissionPolicy(RecordPermissionPolicy):
     can_set_content_files = [
         AllowedByActionRestrictByManageableLibrary(
             create_action, get_library_pid),
-        SystemProcess(),
+        SystemProcess()
     ]
     # files container
     can_read_files = [AllowedByAction(read_action), SystemProcess()]