Merge a58b584 into 36bb490

.editorconfig

@@ -35,6 +35,7 @@ charset = utf-8
 [*.py]
 indent_size = 4
 # isort plugin configuration
+known_standard_library=enum
 known_first_party = invenio_archivematica
 multi_line_output = 2
 default_section = THIRDPARTY

.travis.yml

@@ -55,6 +55,8 @@ before_install:
 install:
   - "travis_retry pip install -r .travis-${REQUIREMENTS}-requirements.txt"
   - "travis_retry pip install -e .[all]"
+# TODO REMOVE WHEN SIPTORE HAS BEEN RELEASED
+  - "travis_retry pip install -U git+https://github.com/inveniosoftware/invenio-sipstore.git#egg=invenio-sipstore"
 
 script:
   - "./run-tests.sh"

AUTHORS.rst

@@ -27,3 +27,4 @@ Authors
 Invenio 3 module to connect Invenio to Archivematica
 
 - CERN <info@inveniosoftware.org>
+- Rémi Ducceschi <remi.ducceschi@gmail.com>

README.rst

@@ -48,5 +48,12 @@ Archivematica is an open-source software used to create digital archives. The
 goal of this module is to create a link between Invenio and Archivematica, so
 Invenio can send records to Archivematica to archive them.
 
+Configuration
+-------------
+
+- setup the Invenio URL in the automation-tools
+- setup the Archivematica URL in the invenio-archivematica config
+- setup the spaces and locations in Archivematica dashboards / storage
+
 Further documentation is available on
 https://invenio-archivematica.readthedocs.io/

docs/api.rst

@@ -28,6 +28,18 @@ API Docs
 .. automodule:: invenio_archivematica.ext
    :members:
 
+API
+---
+
+.. automodule:: invenio_archivematica.api
+    :members:
+
+Models
+------
+
+.. automodule:: invenio_archivematica.models
+    :members:
+
 Views
 -----
 

docs/conf.py

@@ -51,8 +51,12 @@
     'sphinx.ext.doctest',
     'sphinx.ext.intersphinx',
     'sphinx.ext.viewcode',
+    'celery.contrib.sphinx',
 ]
 
+# Remove default Celery sphinx task prefix
+celery_task_prefix = ''
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -338,6 +342,8 @@
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {
     'python': ('https://docs.python.org/', None),
+    'invenio_sipstore': ('https://invenio-sipstore.readthedocs.io/en/latest/', None),
+    'flask': ('http://flask.pocoo.org/docs/0.12/', None),
     # TODO: Configure external documentation references, eg:
     # 'Flask-Admin': ('https://flask-admin.readthedocs.io/en/latest/', None),
 }

docs/usage.rst

@@ -30,5 +30,21 @@ Usage
 Customization
 =============
 
+Factories
+---------
+
 .. automodule:: invenio_archivematica.factories
     :members:
+
+Tasks
+-----
+
+.. automodule:: invenio_archivematica.tasks
+    :members:
+
+.... autotask:: invenio_archivematica.tasks.oais_start_transfer
+.... autotask:: invenio_archivematica.tasks.oais_process_transfer
+.... autotask:: invenio_archivematica.tasks.oais_finish_transfer
+.... autotask:: invenio_archivematica.tasks.oais_fail_transfer
+.... autotask:: invenio_archivematica.tasks.archive_new_sips
+

examples/app-setup.sh

@@ -13,3 +13,6 @@ export FLASK_APP=app.py
 
 # Setup app
 mkdir $DIR/instance
+# Create database
+flask db init
+flask db create

examples/app-teardown.sh

@@ -12,4 +12,5 @@ cd $DIR
 export FLASK_APP=app.py
 
 # Teardown app
+flask db destroy --yes-i-know
 [ -e "$DIR/instance" ] && rm -Rf $DIR/instance

examples/app.py

@@ -56,12 +56,44 @@
 
 from __future__ import absolute_import, print_function
 
+import os
+
 from flask import Flask
 from flask_babelex import Babel
+from invenio_access import InvenioAccess
+from invenio_accounts import InvenioAccounts
+from invenio_db import InvenioDB
+from invenio_files_rest import InvenioFilesREST
+from invenio_pidstore import InvenioPIDStore
+from invenio_rest import InvenioREST
+from invenio_sipstore import InvenioSIPStore
 
 from invenio_archivematica import InvenioArchivematica
+from invenio_archivematica.views import b
 
 # Create Flask application
 app = Flask(__name__)
+app.config.update(dict(
+    BROKER_URL='redis://',
+    CELERY_RESULT_BACKEND='redis://',
+    REST_ENABLE_CORS=True,
+    SECRET_KEY='CHANGEME',
+    SIPSTORE_AGENT_JSONSCHEMA_ENABLED=False,
+    SQLALCHEMY_ECHO=False,
+    SQLALCHEMY_DATABASE_URI=os.environ.get(
+        'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'
+    ),
+    SQLALCHEMY_TRACK_MODIFICATIONS=True,
+))
+
 Babel(app)
+InvenioAccess(app)
+InvenioAccounts(app)
 InvenioArchivematica(app)
+InvenioDB(app)
+InvenioFilesREST(app)
+InvenioPIDStore(app)
+InvenioREST(app)
+InvenioSIPStore(app)
+
+app.register_blueprint(b)

invenio_archivematica/admin.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2017 CERN.
+#
+# Invenio is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+"""Admin views fot the table Archive."""
+
+from flask_admin.contrib.sqla import ModelView
+
+from invenio_archivematica.models import Archive
+
+
+class ArchiveModelView(ModelView):
+    """ModelView for the Archive table."""
+
+    can_create = False
+    can_edit = False
+    can_delete = False
+    can_view_details = True
+    column_display_all_relations = True
+    column_list = (
+        'sip_id', 'sip.created', 'status', 'accession_id', 'archivematica_id'
+    )
+    column_labels = {
+        'sip_id': 'ID of the SIP',
+        'sip.created': 'Last update',
+        'status': 'Status',
+        'accession_id': 'AIP Accession ID',
+        'archivematica_id': 'Archivematica ID'
+    }
+    page_size = 25
+
+
+archive_adminview = dict(
+    modelview=ArchiveModelView,
+    model=Archive,
+    name='Archive',
+    category='Records')

invenio_archivematica/api.py

@@ -24,25 +24,91 @@
 
 """API for Invenio 3 module to connect Invenio to Archivematica."""
 
-from flask import current_app
-from invenio_pidstore.resolver import Resolver
-from invenio_records.api import Record
+from invenio_archivematica.models import ArchiveStatus
+from invenio_archivematica.tasks import oais_fail_transfer, \
+    oais_finish_transfer, oais_process_transfer, oais_start_transfer
 
 
-def create_accessioned_id(record_pid, pid_type):
-    """Create an accessioned ID to store the record in Archivematica.
+def start_transfer(sip, accession_id, archivematica_id=None):
+    """Start the archive process for a sip.
 
-    :param record_pid: the PID of the record
-    :type record_pid: str
-    :param pid_type: the type of the PID ('recid'...)
-    :type pid_type: str
-    :returns: the created ID
-    :rtype: str
+    Start the transfer of the sip in asynchronous mode. See
+    :py:mod:`invenio_archivematica.tasks`
+
+    :param sip: the sip to archive
+    :type sip: :py:class:`invenio_sipstore.api.SIP`
+    :param str accession_id: the accessioned ID in archivematica. You can
+        compute it from
+        :py:func:`invenio_archivematica.factories.create_accession_id`
+    :param str archivematica_id: the ID in Archivematica
+    """
+    oais_start_transfer.delay(str(sip.id), accession_id, archivematica_id)
+
+
+def process_transfer(sip, accession_id='', archivematica_id=None):
+    """Create the archive for a sip.
+
+    Process the transfer of the sip in asynchronous mode. See
+    :py:mod:`invenio_archivematica.tasks`
+
+    :param sip: the sip to archive
+    :type sip: :py:class:`invenio_sipstore.api.SIP`
+    :param str accession_id: the accession_id
+    :param str archivematica_id: the ID of the AIP in Archivematica
     """
-    resolver = Resolver(pid_type=pid_type, getter=Record.get_record)
-    pid, record = resolver.resolve(record_pid)
-    return "{service}-{pid_type}-{pid}-{version}".format(
-        service=current_app.config['ARCHIVEMATICA_ORGANIZATION_NAME'],
-        pid_type=pid_type,
-        pid=record_pid,
-        version=record.revision_id)
+    oais_process_transfer(str(sip.id), accession_id, archivematica_id)
+
+
+def process_aip(sip, accession_id='', archivematica_id=None):
+    """Create the archive for a sip.
+
+    Process the aip of the sip in asynchronous mode. See
+    :py:mod:`invenio_archivematica.tasks`
+
+    :param sip: the sip to archive
+    :type sip: :py:class:`invenio_sipstore.api.SIP`
+    :param str accession_id: the accession_id
+    :param str archivematica_id: the ID of the AIP in Archivematica
+    """
+    oais_process_aip(str(sip.id), accession_id, archivematica_id)
+
+
+def finish_transfer(sip, accession_id='', archivematica_id=None):
+    """Finish the archive process for a sip.
+
+    Finish the transfer of the sip in asynchronous mode. See
+    :py:mod:`invenio_archivematica.tasks`
+
+    :param sip: the sip to archive
+    :type sip: :py:class:`invenio_sipstore.api.SIP`
+    :param str accession_id: the accession_id
+    :param str archivematica_id: the ID of the created AIP in Archivematica
+    """
+    oais_finish_transfer(str(sip.id), accession_id, archivematica_id)
+
+
+def fail_transfer(sip, accession_id='', archivematica_id=None):
+    """Fail the archive process for a sip.
+
+    Fail the transfer of the sip in asynchronous mode. See
+    :py:mod:`invenio_archivematica.tasks`
+
+    :param sip: the sip to archive
+    :type sip: :py:class:`invenio_sipstore.api.SIP`
+    :param str accession_id: the accession_id
+    :param str archivematica_id: the ID of the created AIP in Archivematica
+    """
+    oais_fail_transfer(str(sip.id), accession_id, archivematica_id)
+
+
+change_status_func = {
+    ArchiveStatus.NEW: start_transfer,
+    ArchiveStatus.WAITING: start_transfer,
+    ArchiveStatus.PROCESSING_TRANSFER: process_transfer,
+    ArchiveStatus.PROCESSING_AIP: process_aip,
+    ArchiveStatus.REGISTERED: finish_transfer,
+    ArchiveStatus.FAILED: fail_transfer,
+    ArchiveStatus.IGNORED: None,
+    ArchiveStatus.DELETED: None
+}
+"""Dictionary that maps status to functions used to change the status."""

invenio_archivematica/config.py

@@ -27,14 +27,21 @@
 ARCHIVEMATICA_BASE_TEMPLATE = 'invenio_archivematica/base.html'
 """Default base template for the demo page."""
 
-ARCHIVEMATICA_TRANSFER_FACTORY = 'invenio_archivematica.factories.transfer'
+ARCHIVEMATICA_TRANSFER_FACTORY = 'invenio_archivematica.factories.transfer_cp'
 """The factory to do the transfers of files to the dashboard.
 
 See :py:func:`invenio_archivematica.factories.transfer_cp`
 and :py:data:`invenio_archivematica.config.ARCHIVEMATICA_TRANSFER_FOLDER`
 for more information.
 """
 
+ARCHIVEMATICA_ISARCHIVABLE_FACTORY = 'invenio_archivematica.' \
+                                     'factories.is_archivable_default'
+"""The factory that is used to know if the sip should be archived or not.
+
+See :py:func:`invenio_archivematica.factories.is_archivable_default`.
+"""
+
 ARCHIVEMATICA_ORGANIZATION_NAME = 'CERN'
 """Organization name setup in Archivematica's dashboard."""
 
@@ -46,3 +53,21 @@
 :py:func:`invenio_archivematica.factories.transfer_cp` and
 :py:data:`invenio_archivematica.config.ARCHIVEMATICA_TRANSFER_FACTORY`.
 """
+
+ARCHIVEMATICA_DASHBOARD_URL = 'http://localhost:81'
+"""The URL to Archivematica Dashboard."""
+
+ARCHIVEMATICA_DASHBOARD_USER = 'invenio'
+"""The user to connect to Archivematica Dashboard."""
+
+ARCHIVEMATICA_DASHBOARD_API_KEY = 'change me'
+"""The API key to use with the user above."""
+
+ARCHIVEMATICA_STORAGE_URL = 'http://localhost:8001'
+"""The URL to Archivematica Storage."""
+
+ARCHIVEMATICA_STORAGE_USER = 'invenio'
+"""The user to connect to Archivematica Storage."""
+
+ARCHIVEMATICA_STORAGE_API_KEY = 'change me'
+"""The API key to use with the user above."""