models: per bucket file size limits

* INCOMPATIBLE Changes the file size limiter interface.

* NEW Adds a per bucket maximum file size limit.

* File size limits are now checked while writing the object.

Signed-off-by: Sami Hiltunen <sami.mikael.hiltunen@cern.ch>

AUTHORS.rst

@@ -1,6 +1,6 @@
 ..
     This file is part of Invenio.
-    Copyright (C) 2015 CERN.
+    Copyright (C) 2015, 2016 CERN.
 
     Invenio is free software; you can redistribute it
     and/or modify it under the terms of the GNU General Public License as
@@ -27,7 +27,8 @@ Authors
 
 Files download/upload REST API similar to S3 for Invenio.
 
+- Jiri Kuncar <jiri.kuncar@cern.ch>
 - Jose Benito Gonzalez Lopez <jose.benito.gonzalez@cern.ch>
 - Lars Holm Nielsen <lars.holm.nielsen@cern.ch>
 - Nicolas Harraudeau <nicolas.harraudeau@cern.ch>
-- Jiri Kuncar <jiri.kuncar@cern.ch>
+- Sami Hiltunen <sami.mikael.hiltunen@cern.ch>

invenio_files_rest/__init__.py

@@ -28,5 +28,6 @@
 
 from .ext import InvenioFilesREST
 from .version import __version__
+from .views import current_files_rest
 
-__all__ = ('__version__', 'InvenioFilesREST')
+__all__ = ('__version__', 'current_files_rest', 'InvenioFilesREST', )

invenio_files_rest/config.py

@@ -38,10 +38,20 @@
 FILES_REST_DEFAULT_STORAGE_CLASS = 'S'
 """Default storage class."""
 
-FILES_REST_STORAGE_FACTORY = None
+FILES_REST_DEFAULT_QUOTA_SIZE = None
+"""Default quota size for a bucket in bytes."""
+
+FILES_REST_DEFAULT_MAX_FILE_SIZE = None
+"""Default maximum file size for a bucket in bytes."""
+
+FILES_REST_SIZE_LIMITERS = 'invenio_files_rest.limiters.file_size_limiters'
+"""Import path of file size limiters factory."""
+
+FILES_REST_STORAGE_FACTORY = 'invenio_files_rest.storage.pyfs_storage_factory'
 """Import path of factory used to create a storage instance."""
 
-FILES_REST_PERMISSION_FACTORY = None
+FILES_REST_PERMISSION_FACTORY = \
+    'invenio_files_rest.permissions.permission_factory'
 """Import path of permission factory."""
 
 FILES_REST_OBJECT_KEY_MAX_LEN = 255

invenio_files_rest/errors.py

@@ -26,6 +26,8 @@
 
 from __future__ import absolute_import, print_function
 
+from invenio_rest.errors import RESTException
+
 
 class FilesException(Exception):
     """Base exception for all errors ."""
@@ -44,4 +46,10 @@ class FileInstanceAlreadySetError(FilesException):
 
 
 class InvalidOperationError(FilesException):
-    """Exception raise when an invalid operation is performed."""
+    """Exception raised when an invalid operation is performed."""
+
+
+class FileSizeError(RESTException):
+    """Exception raised when a file larger than allowed."""
+
+    code = 400

invenio_files_rest/ext.py

@@ -31,7 +31,6 @@
 
 from . import config
 from .cli import files as files_cmd
-from .storage import pyfs_storage_factory
 
 
 class _FilesRESTState(object):
@@ -58,43 +57,30 @@ def record_file_factory(self):
     @cached_property
     def storage_factory(self):
         """Load default storage factory."""
-        imp = self.app.config.get("FILES_REST_STORAGE_FACTORY")
-        return import_string(imp) if imp else pyfs_storage_factory
+        return import_string(self.app.config.get('FILES_REST_STORAGE_FACTORY'))
 
     @cached_property
     def permission_factory(self):
         """Load default permission factory."""
-        imp = self.app.config.get("FILES_REST_PERMISSION_FACTORY")
-        if imp:
-            return import_string(imp)
-        else:
-            from invenio_files_rest.permissions import permission_factory
-            return permission_factory
+        return import_string(
+            self.app.config.get('FILES_REST_PERMISSION_FACTORY'))
 
     @cached_property
-    def file_size_limiter(self):
+    def file_size_limiters(self):
         r"""Load the file size limiter.
 
-        The file size limiter is a function used to get the maximum size a file
-        can have. This function can use anything to decide this maximum size,
-        example: bucket quota, user quota, custom limit.
+        The file size limiter is a function used to get the file size limiters.
+        This function can use anything to limit the file size, for example:
+        bucket quota, user quota, custom limit.
         Its prototype is:
-            py::function: limiter(bucket=None\
-                ) -> (size limit: int, reason: str)
 
-        The `reason` is the message displayed to the user when the limit is
-        exceeded.
-        The `size limit` and `reason` can be None if there is no limit.
+            py::function: limiter(bucket=None\
+                ) -> [FileSizeLimit, FileSizeLimit, ...]
 
-        This function is used by the REST API and any other file creation
-        input.
+        An empty list should be returned if there should be no limit. The
+        lowest limit will be used.
         """
-        imp = self.app.config.get("FILES_REST_FILE_SIZE_LIMITER")
-        if imp:
-            return import_string(imp)
-        else:
-            from invenio_files_rest.helpers import file_size_limiter
-            return file_size_limiter
+        return import_string(self.app.config.get('FILES_REST_SIZE_LIMITERS'))
 
 
 class InvenioFilesREST(object):

invenio_files_rest/helpers.py

@@ -85,15 +85,6 @@ def send_stream(stream, filename, size, mtime, mimetype=None, restricted=False,
     return rv
 
 
-def file_size_limiter(bucket):
-    """Retrieve the internal quota from the provided bucket."""
-    if bucket.quota_size:
-        return (bucket.quota_size - bucket.size,
-                'Bucket quota is {0} bytes. {1} bytes are currently '
-                'used.'.format(bucket.quota_size, bucket.size))
-    return (None, None)
-
-
 def compute_md5_checksum(src, chunk_size=None, progress_callback=None):
     """Helper method to compute checksum from a stream.
 

invenio_files_rest/limiters.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2016 CERN.
+#
+# Invenio is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+"""File size limiting functionality for Invenio-Files-REST."""
+
+from __future__ import absolute_import, print_function
+
+
+def file_size_limiters(bucket):
+    """Default file size limiters."""
+    return [
+        FileSizeLimit(
+            bucket.quota_left,
+            'Bucket quota exceeded.',
+        ),
+        FileSizeLimit(
+            bucket.max_file_size,
+            'Maximum file size exceeded.',
+        ),
+    ]
+
+
+class FileSizeLimit(object):
+    """File size limiter."""
+
+    not_implemented_error = NotImplementedError(
+        'FileSizeLimit supports only comparisons with integers and other '
+        'FileSizeLimits.')
+
+    def __init__(self, limit, reason):
+        """Instantiate a new file size limit."""
+        self.limit = limit
+        self.reason = reason
+
+    def __lt__(self, other):
+        """Check if this limit is less than the other one."""
+        if isinstance(other, int):
+            return self.limit < other
+        elif isinstance(other, FileSizeLimit):
+            return self.limit < other.limit
+        raise not_implemented_error
+
+    def __gt__(self, other):
+        """Check if this limit is greater than the other one."""
+        if isinstance(other, int):
+            return self.limit > other
+        elif isinstance(other, FileSizeLimit):
+            return self.limit > other.limit
+        raise not_implemented_error
+
+    def __eq__(self, other):
+        """Check for equality."""
+        if isinstance(other, int):
+            return self.limit == other
+        elif isinstance(other, FileSizeLimit):
+            return self.limit == other.limit
+        raise not_implemented_error

invenio_files_rest/models.py

@@ -62,6 +62,8 @@
 from sqlalchemy_utils.types import UUIDType
 
 from .errors import FileInstanceAlreadySetError, InvalidOperationError
+from .limiters import FileSizeLimit
+from .proxies import current_files_rest
 
 slug_pattern = re.compile('^[a-z][a-z0-9-]+$')
 
@@ -192,13 +194,19 @@ class Bucket(db.Model, Timestamp):
     inside the bucket.
     """
 
-    quota_size = db.Column(db.BigInteger, nullable=True)
-    """Quota size of bucket. Used only by the file_size_limiter.
+    quota_size = db.Column(
+        db.BigInteger,
+        nullable=True,
+        default=lambda: current_app.config['FILES_REST_DEFAULT_QUOTA_SIZE']
+    )
+    """Quota size of bucket."""
 
-    Note, don't use this attribute directly. It MAY be used to store
-    the actual quota size for this bucket. Change the file_size_limiter if
-    you want to filter accepted files based on their size.
-    """
+    max_file_size = db.Column(
+        db.BigInteger,
+        nullable=True,
+        default=lambda: current_app.config['FILES_REST_DEFAULT_MAX_FILE_SIZE']
+    )
+    """Maximum size of a single file in the bucket."""
 
     locked = db.Column(db.Boolean, default=False, nullable=False)
     """Is bucket locked?"""
@@ -213,6 +221,12 @@ def __repr__(self):
         """Return representation of location."""
         return str(self.id)
 
+    @property
+    def quota_left(self):
+        """Get how much space is left in the bucket."""
+        if self.quota_size:
+            return max(self.quota_size - self.size, 0)
+
     @validates('default_storage_class')
     def validate_storage_class(self, key, default_storage_class):
         """Validate storage class."""
@@ -453,8 +467,7 @@ def storage(self, **kwargs):
 
         :returns: Storage interface.
         """
-        return current_app.extensions['invenio-files-rest'].storage_factory(
-            fileinstance=self, **kwargs)
+        return current_files_rest.storage_factory(fileinstance=self, **kwargs)
 
     def verify_checksum(self, progress_callback=None, **kwargs):
         """Verify checksum of file instance."""
@@ -465,7 +478,7 @@ def verify_checksum(self, progress_callback=None, **kwargs):
             self.last_check_at = datetime.utcnow()
         return self.last_check
 
-    def set_contents(self, stream, chunk_size=None,
+    def set_contents(self, stream, chunk_size=None, size=None, size_limit=None,
                      progress_callback=None, **kwargs):
         """Save contents of stream to this file.
 
@@ -477,8 +490,8 @@ def set_contents(self, stream, chunk_size=None,
             raise ValueError('File instance is not writable.')
         self.set_uri(
             *self.storage(**kwargs).save(
-                stream, chunk_size=chunk_size,
-                progress_callback=progress_callback))
+                stream, chunk_size=chunk_size, size=size,
+                size_limit=size_limit, progress_callback=progress_callback))
 
     def copy_contents(self, fileinstance, progress_callback=None, **kwargs):
         """Copy this file instance into another file instance."""
@@ -599,7 +612,7 @@ def is_deleted(self):
         """Determine if object version is a delete marker."""
         return self.file_id is None
 
-    def set_contents(self, stream, size=None, chunk_size=None,
+    def set_contents(self, stream, size_limit=None, size=None, chunk_size=None,
                      progress_callback=None):
         """Save contents of stream to file instance.
 
@@ -614,9 +627,17 @@ def set_contents(self, stream, size=None, chunk_size=None,
         if self.file_id is not None:
             raise FileInstanceAlreadySetError()
 
+        if size_limit is None:
+            limits = [
+                lim for lim in current_files_rest.file_size_limiters(
+                    self.bucket)
+                if lim.limit is not None
+            ]
+            size_limit = min(limits) if limits else None
+
         self.file = FileInstance.create()
         self.file.set_contents(
-            stream, size=size, chunk_size=chunk_size,
+            stream, size_limit=size_limit, size=size, chunk_size=chunk_size,
             progress_callback=progress_callback, objectversion=self)
 
         self.bucket.size += self.file.size
@@ -753,18 +774,18 @@ def get(cls, bucket, key, version_id=None):
         """
         bucket_id = bucket.id if isinstance(bucket, Bucket) else bucket
 
-        args = [
+        filters = [
             cls.bucket_id == bucket_id,
             cls.key == key,
         ]
 
         if version_id:
-            args.append(cls.version_id == version_id)
+            filters.append(cls.version_id == version_id)
         else:
-            args.append(cls.is_head.is_(True))
-            args.append(cls.file_id.isnot(None))
+            filters.append(cls.is_head.is_(True))
+            filters.append(cls.file_id.isnot(None))
 
-        return cls.query.filter(*args).one_or_none()
+        return cls.query.filter(*filters).one_or_none()
 
     @classmethod
     def get_versions(cls, bucket, key):
@@ -775,12 +796,12 @@ def get_versions(cls, bucket, key):
         """
         bucket_id = bucket.id if isinstance(bucket, Bucket) else bucket
 
-        args = [
+        filters = [
             cls.bucket_id == bucket_id,
             cls.key == key,
         ]
 
-        return cls.query.filter(*args).order_by(cls.key, cls.created.desc())
+        return cls.query.filter(*filters).order_by(cls.key, cls.created.desc())
 
     @classmethod
     def delete(cls, bucket, key):
@@ -803,15 +824,15 @@ def get_by_bucket(cls, bucket, versions=False):
         """Return query that fetches all the objects in a bucket."""
         bucket_id = bucket.id if isinstance(bucket, Bucket) else bucket
 
-        args = [
+        filters = [
             cls.bucket_id == bucket_id,
         ]
 
         if not versions:
-            args.append(cls.file_id.isnot(None))
-            args.append(cls.is_head.is_(True))
+            filters.append(cls.file_id.isnot(None))
+            filters.append(cls.is_head.is_(True))
 
-        return cls.query.filter(*args).order_by(cls.key, cls.created.desc())
+        return cls.query.filter(*filters).order_by(cls.key, cls.created.desc())
 
     @classmethod
     def relink_all(cls, old_file, new_file):