Added metadata method

CHANGELOG.rst

@@ -4,7 +4,7 @@ Changelog
 Current
 -------
 
-- Nothing yet
+- Added `metadata` method to `Storage` to retrieve file metadata
 
 0.4.1 (2017-06-24)
 ------------------

flask_fs/backends/gridfs.py

@@ -11,6 +11,7 @@
 from gridfs import GridFS
 from pymongo import MongoClient
 
+from flask_fs import files
 
 from . import BaseBackend
 
@@ -67,3 +68,12 @@ def list_files(self):
     def serve(self, filename):
         file = self.fs.get_last_version(filename)
         return send_file(file, mimetype=file.content_type)
+
+    def metadata(self, filename):
+        f = self.fs.get_last_version(filename)
+        return {
+            'checksum': 'md5:{0}'.format(f.md5),
+            'size': f.length,
+            'mime': f.content_type or files.mime(filename),
+            'modified': f.upload_date,
+        }

flask_fs/backends/local.py

@@ -1,21 +1,39 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import hashlib
 import io
 import logging
 import os
 
+from datetime import datetime
 from shutil import copyfileobj
 
 from flask import current_app, send_from_directory
 from werkzeug import cached_property
 from werkzeug.datastructures import FileStorage
 
+from flask_fs import files
+
 from . import BaseBackend
 
 log = logging.getLogger(__name__)
 
 
+CHUNK_SIZE = 2 ** 16
+
+
+def sha1(file):
+    hasher = hashlib.sha1()
+    blk_size_to_read = hasher.block_size * CHUNK_SIZE
+    while (True):
+        read_data = file.read(blk_size_to_read)
+        if not read_data:
+            break
+        hasher.update(read_data)
+    return hasher.hexdigest()
+
+
 class LocalBackend(BaseBackend):
     '''
     A local file system storage
@@ -87,3 +105,15 @@ def path(self, filename):
     def serve(self, filename):
         '''Serve files for storages with direct file access'''
         return send_from_directory(self.root, filename)
+
+    def metadata(self, filename):
+        '''Fetch all availabe metadata'''
+        dest = self.path(filename)
+        with open(dest, 'rb', buffering=0) as f:
+            checksum = 'sha1:{0}'.format(sha1(f))
+        return {
+            'checksum': checksum,
+            'size': os.path.getsize(dest),
+            'mime': files.mime(filename),
+            'modified': datetime.fromtimestamp(os.path.getmtime(dest)),
+        }

flask_fs/backends/s3.py

@@ -78,6 +78,17 @@ def list_files(self):
         for f in self.bucket.objects.all():
             yield f.key
 
+    def metadata(self, filename):
+        '''Fetch all availabe metadata'''
+        obj = self.bucket.Object(filename)
+        checksum = 'md5:{0}'.format(obj.e_tag[1:-1])
+        return {
+            'checksum': checksum,
+            'size': obj.content_length,
+            'mime': obj.content_type,
+            'modified': obj.last_modified,
+        }
+
     # def serve(self, filename):
     #     file = self.fs.get_last_version(filename)
     #     return send_file(file, mimetype=file.content_type)

flask_fs/backends/swift.py

@@ -5,6 +5,7 @@
 import logging
 
 from contextlib import contextmanager
+from dateutil import parser
 
 import swiftclient
 
@@ -64,3 +65,12 @@ def list_files(self):
         headers, items = self.conn.get_container(self.name)
         for i in items:
             yield i['name']
+
+    def metadata(self, filename):
+        data = self.conn.head_object(self.name, filename)
+        return {
+            'checksum': 'md5:{0}'.format(data['etag']),
+            'size': int(data['content-length']),
+            'mime': data['content-type'],
+            'modified': parser.parse(data['last-modified']),
+        }

flask_fs/files.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import mimetypes
 import os.path
 
 __all__ = (
@@ -67,6 +68,13 @@ def lower_extension(filename):
     return filename
 
 
+def mime(filename):
+    '''
+    A basic helper to guess mime type from a filename or url
+    '''
+    return mimetypes.guess_type(filename)[0]
+
+
 class All(object):
     '''
     This type can be used to allow all extensions.

flask_fs/storage.py

@@ -307,6 +307,22 @@ def list_files(self):
         '''
         return self.backend.list_files()
 
+    def metadata(self, filename):
+        '''
+        Get some metadata for a given file.
+
+        Can vary from a backend to another but some are always present:
+        - `filename`: the base filename (without the path/prefix)
+        - `url`: the file public URL
+        - `checksum`: a checksum expressed in the form `algo:hash` 
+        - 'mime': the mime type
+        - `modified`: the last modification date
+        '''
+        metadata = self.backend.metadata(filename)
+        metadata['filename'] = os.path.basename(filename)
+        metadata['url'] = self.url(filename, external=True)
+        return metadata
+
     def __contains__(self, value):
         return self.exists(value)
 

requirements/install.pip

@@ -1,2 +1,3 @@
 flask
+python-dateutil
 six

tests/test_backend_mixin.py

@@ -1,8 +1,11 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import hashlib
 import six
 
+from datetime import datetime
+
 
 class BackendTestCase(object):
 
@@ -141,3 +144,15 @@ def test_list_files(self, faker, utils):
             self.put_file(f, content)
 
         assert sorted(list(self.backend.list_files())) == ['first.test', 'second.test']
+
+    def test_metadata(self, app, faker):
+        content = six.text_type(faker.sentence())
+        hasher = getattr(hashlib, self.hasher)
+        hashed = hasher(content.encode('utf8')).hexdigest()
+        self.put_file('file.txt', content)
+
+        metadata = self.backend.metadata('file.txt')
+        assert metadata['checksum'] == '{0}:{1}'.format(self.hasher, hashed)
+        assert metadata['size'] == len(content)
+        assert metadata['mime'] == 'text/plain'
+        assert isinstance(metadata['modified'], datetime)

tests/test_gridfs_backend.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-from pymongo import MongoClient
 from gridfs import GridFS
+from pymongo import MongoClient
 
 from .test_backend_mixin import BackendTestCase
 
@@ -15,6 +15,8 @@
 
 
 class GridFsBackendTest(BackendTestCase):
+    hasher = 'md5'
+
     @pytest.fixture(autouse=True)
     def setup(self):
         self.client = MongoClient()

tests/test_local_backend.py

@@ -10,6 +10,8 @@
 
 
 class LocalBackendTest(BackendTestCase):
+    hasher = 'sha1'
+
     @pytest.fixture(autouse=True)
     def setup(self, tmpdir):
         self.test_dir = tmpdir

tests/test_s3_backend.py

@@ -27,6 +27,8 @@
 
 
 class S3BackendTest(BackendTestCase):
+    hasher = 'md5'
+
     @pytest.fixture(autouse=True)
     def setup(self):
         self.session = boto3.session.Session()

tests/test_storage.py

@@ -419,3 +419,18 @@ def test_list_files(app, mock_backend):
     app.configure(storage)
 
     assert storage.list_files() == ['one.txt']
+
+
+def test_metadata(app, mock_backend):
+    storage = fs.Storage('test')
+    app.configure(storage)
+
+    backend = mock_backend.return_value
+    backend.metadata.return_value = {}
+
+    url = url_for('fs.get_file', filename='file.test', fs=storage.name, _external=True)
+
+    metadata = storage.metadata('file.test')
+    assert metadata['filename'] == 'file.test'
+    assert metadata['url'] == url
+    backend.metadata.assert_called_with('file.test')

tests/test_swift_backend.py

@@ -16,6 +16,8 @@
 
 
 class SwiftBackendTest(BackendTestCase):
+    hasher = 'md5'
+
     @pytest.fixture(autouse=True)
     def setup(self):
         self.conn = swiftclient.Connection(