deposit: fixes for JSON-based records

* Adds invenio-records the devel requirements.

* INCOMPATIBLE Reworks the default (i.e. example/simple) workflow for
  deposit. SIPs are now sealed and dumped after we collected all form
  data and before make ANY further modification to the deposit. This
  also holds for adding the default collection information to the
  deposit. Further analysis and work is required to make this process
  standard compliant (e.g. splitting SIP, DIP, ...). Dumped SIPs are
  now in JSON (was XML).

Signed-off-by: Marco Neumann <marco@crepererum.net>

invenio/modules/deposit/helpers.py

@@ -80,14 +80,14 @@ def deposition_record(record, form_classes, pre_process_load=None,
     )
 
 
-def make_record(values, is_dump=True):
+def make_record(values, is_dump=None):
     """
     Export recjson from drafts
     """
-    if is_dump:
-        record = Record(json=values, master_format='marc')
-    else:
-        record = Record(master_format='marc')
-        for k, v in six.iteritems(values):
-            record[k] = v
-    return record
+    if is_dump is not None:
+        import warnings
+        warnings.warn(
+            "Usage of `is_dump` parameter is deprecated and has no effect.",
+            category=UserWarning
+        )
+    return Record(data=values)

invenio/modules/deposit/tasks.py

@@ -21,6 +21,7 @@
 
 from __future__ import print_function
 
+import json
 import os
 import dictdiffer
 import dateutil.parser
@@ -31,7 +32,7 @@
 from flask_login import current_user
 from functools import wraps
 
-from invenio_records.api import get_record
+from invenio_records.api import create_record, get_record
 from invenio.modules.editor.models import HstRECORD
 from invenio.modules.deposit.models import Deposition, Agent, \
     DepositionDraftCacheManager
@@ -347,7 +348,7 @@ def create_recid():
     @wraps(create_recid)
     def _create_recid(obj, dummy_eng):
         d = Deposition(obj)
-        sip = d.get_latest_sip(sealed=False)
+        sip = d.get_latest_sip(sealed=True)
         if sip is None:
             raise Exception("No submission information package found.")
 
@@ -448,7 +449,7 @@ def process_sip_metadata(processor=None):
     @wraps(process_sip_metadata)
     def _process_sip(obj, dummy_eng):
         d = Deposition(obj)
-        metadata = d.get_latest_sip(sealed=False).metadata
+        metadata = d.get_latest_sip(sealed=True).metadata
 
         if processor is not None:
             processor(d, metadata)
@@ -459,19 +460,6 @@ def _process_sip(obj, dummy_eng):
     return _process_sip
 
 
-def finalize_record_sip(is_dump=True):
-    """Finalize the SIP by generating the MARC and storing it in the SIP."""
-    @wraps(finalize_record_sip)
-    def _finalize_sip(obj, dummy_eng):
-        d = Deposition(obj)
-        sip = d.get_latest_sip(sealed=False)
-        sip.package = make_record(
-            sip.metadata, is_dump=is_dump
-        ).legacy_export_as_marc()
-        d.update()
-    return _finalize_sip
-
-
 def hold_for_approval():
     """Hold deposition on the Holding Pen for admin approval."""
     @wraps(hold_for_approval)
@@ -483,43 +471,40 @@ def _hold_for_approval(obj, dummy_eng):
     return _hold_for_approval
 
 
-def upload_record_sip():
-    """Generate the record from marc.
-
-    The function requires the marc to be generated,
-    so the function export_marc_from_json must have been called successfully
-    before
-    """
-    @wraps(upload_record_sip)
-    def create(obj, dummy_eng):
+def dump_record_sip():
+    @wraps(dump_record_sip)
+    def dump(obj, dummy_eng):
         #FIXME change share tmp directory
-        from invenio.config import CFG_TMPSHAREDDIR
-        from invenio.legacy.bibsched.bibtask import task_low_level_submission, \
-            bibtask_allocate_sequenceid
         d = Deposition(obj)
 
         sip = d.get_latest_sip(sealed=False)
-        sip.seal()
+        sip.seal() # FIXME now? or later?
 
         tmp_file_fd, tmp_file_path = mkstemp(
             prefix="webdeposit-%s-%s" % (d.id, sip.uuid),
-            suffix='.xml',
-            dir=CFG_TMPSHAREDDIR,
+            suffix='.json',
+            dir=current_app.config.get('CFG_TMPSHAREDDIR'),
         )
 
-        os.write(tmp_file_fd, sip.package)
+        os.write(tmp_file_fd, json.dumps(sip.metadata))
         os.close(tmp_file_fd)
 
-        # Trick to have access to task_sequence_id in subsequent tasks.
-        d.workflow_object.task_sequence_id = bibtask_allocate_sequenceid()
+        sip.task_ids.append(None) # FIXME
 
-        task_id = task_low_level_submission(
-            'bibupload', 'webdeposit',
-            '-r' if 'recid' in sip.metadata else '-i', tmp_file_path,
-            '-I', str(d.workflow_object.task_sequence_id)
-        )
+        d.update()
+    return dump
+
+
+def store_record():
+    @wraps(store_record)
+    def create(obj, dummy_eng):
+        #FIXME change share tmp directory
+        d = Deposition(obj)
+
+        sip = d.get_latest_sip(sealed=True)
+        sip.seal() #FIXME ???
 
-        sip.task_ids.append(task_id)
+        create_record(data=sip.metadata)
 
         d.update()
     return create

invenio/modules/deposit/templates/deposit/completed_base.html

@@ -46,7 +46,7 @@ <h4>{{ _('Preview') }}</h4>
              x_contact='<a href="mailto:{email}">{email}</a>'.format(email=config.CFG_SITE_SUPPORT_EMAIL)|safe) }}
     </small>
     <hr />
-    {{format_record(recID=sip.metadata['recid'], xml_record=sip.package, of='hd')|safe}}
+    {{format_record(sip.metadata, of='hd')|safe}}
 
     </div>
     <div class="col-md-4">

invenio/modules/deposit/types/simplerecord.py

@@ -25,8 +25,8 @@
 from invenio.modules.deposit.tasks import render_form, \
     create_recid, \
     prepare_sip, \
-    finalize_record_sip, \
-    upload_record_sip, \
+    dump_record_sip, \
+    store_record, \
     prefill_draft, \
     process_sip_metadata, \
     hold_for_approval
@@ -44,18 +44,18 @@ class SimpleRecordDeposition(DepositionType):
         # Create the submission information package by merging form data
         # from all drafts (in this case only one draft exists).
         prepare_sip(),
+        # Dump unmodified SIP
+        dump_record_sip(),
         # Process metadata to match your JSONAlchemy record model. This will
         # call process_sip_metadata() on your subclass.
         process_sip_metadata(),
         # Reserve a new record id, so that we can provide proper feedback to
         # user before the record has been uploaded.
         create_recid(),
-        # Generate MARC based on metadata dictionary.
-        finalize_record_sip(is_dump=False),
         # Hold the deposition for admin approval
         hold_for_approval(),
-        # Seal the SIP and write MARCXML file and call bibupload on it
-        upload_record_sip(),
+        # Finally create the record
+        store_record(),
     ]
 
     hold_for_upload = False

requirements-devel.txt

@@ -17,3 +17,4 @@
 -e git+git://github.com/inveniosoftware/flask-registry.git#egg=flask-registry
 -e git+git://github.com/inveniosoftware/flask-iiif.git#egg=flask-iiif
 -e git+git://github.com/inveniosoftware/invenio-query-parser.git#egg=invenio-query-parser
+-e git+git://github.com/inveniosoftware/invenio-records.git#egg=invenio-records