Merge d11c0bc into 13ff0c8

.editorconfig

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Invenio.
-# Copyright (C) 2015 CERN.
+# Copyright (C) 2015, 2016 CERN.
 #
 # Invenio is free software; you can redistribute it
 # and/or modify it under the terms of the GNU General Public License as
@@ -38,7 +38,7 @@ indent_size = 4
 # isort plugin configuration
 known_standard_library = mimetypes
 known_first_party = invenio_files_rest
-known_third_party = invenio_access
+known_third_party = invenio_access, invenio_admin, invenio_accounts, invenio_db
 multi_line_output = 2
 default_section = THIRDPARTY
 

examples/app.py

@@ -45,9 +45,14 @@
 
 .. code-block:: console
 
-   $ cd examples
    $ flask -a app.py --debug run
 
+Run example worker:
+
+.. code-block:: console
+
+   $ celery worker -A app.celery -l info --purge
+
 Administration interface is available on::
 
    http://localhost:5000/admin/
@@ -71,16 +76,18 @@
 from invenio_accounts import InvenioAccounts
 from invenio_accounts.views import blueprint
 from invenio_admin import InvenioAdmin
+from invenio_celery import InvenioCelery
 from invenio_db import InvenioDB, db
 from invenio_rest import InvenioREST
 
 from invenio_files_rest import InvenioFilesREST
-from invenio_files_rest.models import Bucket, FileInstance, Location, Object
+from invenio_files_rest.models import Bucket, FileInstance, Location, \
+    ObjectVersion
 
 # Create Flask application
 app = Flask(__name__)
 app.config.update(dict(
-    CELERY_ALWAYS_EAGER=True,
+    CELERY_ALWAYS_EAGER=False,
     CELERY_CACHE_BACKEND='memory',
     CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
     CELERY_RESULT_BACKEND='cache',
@@ -104,6 +111,8 @@
 
 app.register_blueprint(blueprint)
 
+celery = InvenioCelery(app).celery
+
 
 @app.cli.group()
 def fixtures():
@@ -120,7 +129,7 @@ def files():
     makedirs(d)
 
     # Clear data
-    Object.query.delete()
+    ObjectVersion.query.delete()
     Bucket.query.delete()
     FileInstance.query.delete()
     Location.query.delete()
@@ -133,22 +142,22 @@ def files():
     # Bucket 1
     b1 = Bucket.create(loc)
     for f in ['README.rst', 'LICENSE']:
-        with open(join(srcroot, f), 'r') as fp:
-            Object.create(b1, f, stream=fp)
+        with open(join(srcroot, f), 'rb') as fp:
+            ObjectVersion.create(b1, f, stream=fp)
 
     # Bucket 1
     b2 = Bucket.create(loc)
     k = 'AUTHORS.rst'
-    with open(join(srcroot, 'CHANGES.rst'), 'r') as fp:
-        Object.create(b2, k, stream=fp)
-    with open(join(srcroot, 'AUTHORS.rst'), 'r') as fp:
-        Object.create(b2, k, stream=fp)
+    with open(join(srcroot, 'CHANGES.rst'), 'rb') as fp:
+        ObjectVersion.create(b2, k, stream=fp)
+    with open(join(srcroot, 'AUTHORS.rst'), 'rb') as fp:
+        ObjectVersion.create(b2, k, stream=fp)
 
     k = 'RELEASE-NOTES.rst'
-    with open(join(srcroot, 'RELEASE-NOTES.rst'), 'r') as fp:
-        Object.create(b2, k, stream=fp)
-    with open(join(srcroot, 'CHANGES.rst'), 'r') as fp:
-        Object.create(b2, k, stream=fp)
-    Object.delete(b2.id, k)
+    with open(join(srcroot, 'RELEASE-NOTES.rst'), 'rb') as fp:
+        ObjectVersion.create(b2, k, stream=fp)
+    with open(join(srcroot, 'CHANGES.rst'), 'rb') as fp:
+        ObjectVersion.create(b2, k, stream=fp)
+    ObjectVersion.delete(b2.id, k)
 
     db.session.commit()

invenio_files_rest/admin.py

@@ -21,12 +21,19 @@
 
 from __future__ import absolute_import, print_function
 
-from flask import current_app
+import uuid
+
+from flask import current_app, flash, url_for
+from flask_admin.actions import action
 from flask_admin.contrib.sqla import ModelView
 from flask_wtf import Form
+from invenio_admin.filters import FilterConverter
+from invenio_admin.forms import LazyChoices
+from markupsafe import Markup
 from wtforms.validators import ValidationError
 
 from .models import Bucket, FileInstance, Location, ObjectVersion, slug_pattern
+from .tasks import verify_checksum
 
 
 def _(x):
@@ -40,28 +47,30 @@ def require_slug(form, field):
         raise ValidationError(_("Invalid location name."))
 
 
-class LazyChoices(object):
-    """Lazy form choices."""
-
-    def __init__(self, func):
-        """Initialize lazy choices."""
-        self._func = func
-
-    def __iter__(self):
-        """Iterate over lazy choices."""
-        return iter(self._func())
+def link(text, link_func):
+    """Generate a object formatter for links.."""
+    def object_formatter(v, c, m, p):
+        """Format object view link."""
+        return Markup('<a href="{0}">{1}</a>'.format(
+            link_func(m), text))
+    return object_formatter
 
 
 class LocationModelView(ModelView):
     """ModelView for the locations."""
 
+    filter_converter = FilterConverter()
     can_create = True
     can_edit = True
     can_delete = True
     can_view_details = True
-    column_editable_list = ('default', )
-    column_details_list = ('name', 'uri', 'default', 'created', 'updated', )
-    column_list = ('name', 'uri', 'default', 'created', 'updated', )
+    column_formatters = dict(
+        buckets=link('Buckets', lambda o: url_for(
+            'bucket.index_view', flt2_2=o.name))
+    )
+    column_details_list = (
+        'name', 'uri', 'default', 'created', 'updated', 'buckets')
+    column_list = ('name', 'uri', 'default', 'created', 'updated', 'buckets')
     column_labels = dict(
         id=_('ID'),
         uri=_('URI'),
@@ -80,25 +89,35 @@ class LocationModelView(ModelView):
 class BucketModelView(ModelView):
     """ModelView for the buckets."""
 
+    filter_converter = FilterConverter()
     can_create = True
     can_delete = False
     can_edit = True
     can_view_details = True
+    column_formatters = dict(
+        objects=link('Objects', lambda o: url_for(
+            'objectversion.index_view', flt0_0=o.id, flt1_37=1, sort=1)),
+        object_versions=link('Object Versions', lambda o: url_for(
+            'objectversion.index_view', flt0_0=o.id, flt1_29=1, sort=1)),
+    )
     column_details_list = (
         'id', 'location', 'default_storage_class', 'deleted', 'locked',
-        'created', 'updated', )
+        'created', 'updated', 'objects', 'object_versions'
+    )
     column_list = (
         'id', 'location', 'default_storage_class', 'deleted', 'locked', 'size',
-        'created', 'updated',
+        'created', 'updated', 'objects',
     )
     column_labels = dict(
         id=_('UUID'),
         default_location=_('Location'),
         pid_provider=_('Storage Class'),
     )
     column_filters = (
-        'deleted', 'locked', 'default_location', 'default_storage_class',
-        'created', 'updated')
+        # Change of order affects Location.column_formatters!
+        'location.name', 'default_storage_class', 'deleted', 'locked', 'size',
+        'created', 'updated',
+    )
     column_default_sort = ('updated', True)
     form_base_class = Form
     form_columns = ('location', 'default_storage_class', 'locked', 'deleted', )
@@ -111,52 +130,105 @@ class BucketModelView(ModelView):
 class ObjectModelView(ModelView):
     """ModelView for the objects."""
 
+    filter_converter = FilterConverter()
     can_create = False
     can_edit = False
     can_delete = False
     can_view_details = True
+    column_formatters = dict(
+        file_instance=link('File', lambda o: url_for(
+            'fileinstance.index_view', flt0_0=o.file_id)),
+        versions=link('Versions', lambda o: url_for(
+            'objectversion.index_view',
+            sort=7, desc=1, flt0_0=o.bucket_id, flt1_29=o.key)),
+        bucket_objs=link('Objects', lambda o: url_for(
+            'objectversion.index_view',
+            flt0_0=o.bucket_id, flt1_37=1, sort=1)),
+    )
     column_labels = {
         'version_id': _('Version'),
+        'file_id': _('File UUID'),
         'file.uri': _('URI'),
         'file.size': _('Size'),
         'is_deleted': _('Deleted'),
+        'file.checksum': _('Checksum'),
+        'file.read_only': _('Read only'),
+        'file.storage_class': _('Storage class'),
+        'bucket_objs': _("Objects"),
+        'file_instance': _("File"),
     }
     column_list = (
         'bucket', 'key', 'version_id', 'file.uri', 'is_head', 'is_deleted',
-        'file.size', 'created', 'updated', )
+        'file.size', 'created', 'updated', 'versions', 'bucket_objs',
+        'file_instance')
     column_searchable_list = ('key', )
     column_details_list = (
         'bucket', 'key', 'version_id', 'file_id', 'file.uri', 'file.checksum',
         'file.size', 'file.storage_class', 'is_head', 'is_deleted', 'created',
-        'updated', )
+        'updated', 'file_instance', 'versions')
     column_filters = (
-        'bucket', 'key', 'is_head', 'file.storage_class', 'created', 'updated')
+        # Order affects column_formatters in other model views.
+        'bucket.id', 'bucket.locked', 'bucket.deleted', 'bucket.location.name',
+        'file_id', 'file.checksum', 'file.storage_class', 'file.read_only',
+        'key', 'version_id', 'is_head', 'file.size', 'created', 'updated', )
+    column_sortable_list = (
+        'key', 'file.uri', 'is_head', 'file.size', 'created', 'updated')
     column_default_sort = ('updated', True)
     page_size = 25
 
 
 class FileInstanceModelView(ModelView):
     """ModelView for the objects."""
 
+    filter_converter = FilterConverter()
     can_create = False
     can_edit = False
     can_delete = False
     can_view_details = True
+    column_formatters = dict(
+        objects=link('Objects', lambda o: url_for(
+            'objectversion.index_view', flt3_12=o.id)),
+    )
     column_labels = dict(
+        id=_('ID'),
         uri=_('URI'),
+        last_check=_('Fixity'),
+        last_check_at=_('Checked'),
     )
     column_list = (
         'id', 'uri', 'storage_class', 'size', 'checksum', 'read_only',
-        'created', 'updated', )
+        'last_check', 'last_check_at', 'created', 'updated', 'objects')
     column_searchable_list = ('uri', 'size', 'checksum', )
     column_details_list = (
         'id', 'uri', 'storage_class', 'size', 'checksum', 'read_only',
-        'created', 'updated', )
+        'last_check', 'last_check_at', 'created', 'updated', 'objects', )
     column_filters = (
-        'uri', 'size', 'checksum', 'read_only', 'created', 'updated')
+        'id', 'uri', 'storage_class', 'size', 'checksum', 'read_only',
+        'last_check', 'last_check_at', 'created', 'updated')
     column_default_sort = ('updated', True)
     page_size = 25
 
+    @action('verify_checksum', _('Run fixity check'))
+    def action_verify_checksum(self, ids):
+        """Inactivate users."""
+        try:
+            count = 0
+            for file_id in ids:
+                f = FileInstance.query.filter_by(
+                    id=uuid.UUID(file_id)).one_or_none()
+                if f is None:
+                    raise ValueError(_("Cannot find file instance."))
+                verify_checksum.delay(file_id)
+                count += 1
+            if count > 0:
+                flash(_('Fixity check(s) sent to queue.'), 'success')
+        except Exception as exc:
+            if not self.handle_view_exception(exc):
+                raise
+            current_app.logger.exception(str(exc))  # pragma: no cover
+            flash(_('Failed to run fixity checks.'),
+                  'error')  # pragma: no cover
+
 
 location_adminview = dict(
     modelview=LocationModelView,