Refactored storage to factorize some code

tc_aws/__init__.py

@@ -1,7 +1,6 @@
 # coding: utf-8
-from thumbor.config import Config
-
 
+from thumbor.config import Config
 
 Config.define('STORAGE_BUCKET',                         'thumbor-images',   'S3 bucket for Storage', 'S3')
 Config.define('RESULT_STORAGE_BUCKET',                  'thumbor-result',   'S3 bucket for result Storage', 'S3')

tc_aws/aws/__init__.py

@@ -0,0 +1 @@
+# coding: utf-8

tc_aws/aws/storage.py

@@ -0,0 +1,96 @@
+# coding: utf-8
+
+import calendar
+
+from os.path import join
+
+from datetime import datetime, timedelta
+
+from thumbor.utils import logger
+
+from boto.s3.bucket import Bucket
+from boto.s3.key import Key
+from dateutil.parser import parse as parse_ts
+
+from connection import get_connection
+
+class AwsStorage():
+
+    @property
+    def is_auto_webp(self):
+        return self.context.config.AUTO_WEBP and self.context.request.accepts_webp
+
+    def __init__(self, context, config_prefix):
+        self.config_prefix = config_prefix
+        self.context = context
+        self.storage = self.__get_s3_bucket()
+
+    def _get_config(self, config_key, default = None):
+        return getattr(self.context.config, '%s_%s' % (self.config_prefix, config_key))
+
+    def __get_s3_bucket(self):
+        return Bucket(
+            connection=get_connection(self.context),
+            name=self._get_config('BUCKET')
+        )
+
+    def set(self, bytes, abspath):
+        file_key=Key(self.storage)
+        file_key.key=abspath
+
+        file_key.set_contents_from_string(bytes,
+            encrypt_key=self.context.config.get('S3_STORAGE_SSE', ''),         #TODO: fix config prefix
+            reduced_redundancy=self.context.config.get('S3_STORAGE_RRS', '')   #TODO: fix config prefix
+        )
+
+    def get(self, path):
+        file_abspath = self.normalize_path(path)
+
+        file_key = self.storage.get_key(file_abspath)
+
+        if not file_key or self.is_expired(file_key):
+            logger.debug("[STORAGE] s3 key not found at %s" % file_abspath)
+            return None
+
+        return file_key.read()
+
+    def normalize_path(self, path):
+        path = path.lstrip('/')  # Remove leading '/'
+        path_segments = [self._get_config('AWS_STORAGE_ROOT_PATH'), path]
+
+        if self.is_auto_webp:
+            path_segments.append("webp")
+
+        return join(*path_segments)
+
+    def is_expired(self, key):
+        if key:
+            expire_in_seconds = self._get_config('EXPIRATION_SECONDS')
+
+            # Never expire
+            if expire_in_seconds is None or expire_in_seconds == 0:
+                return False
+
+            timediff = datetime.now() - self.utc_to_local(parse_ts(key.last_modified))
+            return timediff.seconds > self._get_config('EXPIRATION_SECONDS')
+        else:
+            #If our key is bad just say we're expired
+            return True
+
+    def last_updated(self):
+        path = self.context.request.url
+        file_abspath = self.normalize_path(path)
+        file_key = self.storage.get_key(file_abspath)
+
+        if not file_key or self.is_expired(file_key):
+            logger.debug("[RESULT_STORAGE] s3 key not found at %s" % file_abspath)
+            return None
+
+        return self.utc_to_local(parse_ts(file_key.last_modified))
+
+    def utc_to_local(self,utc_dt):
+        # get integer timestamp to avoid precision lost
+        timestamp = calendar.timegm(utc_dt.timetuple())
+        local_dt = datetime.fromtimestamp(timestamp)
+        assert utc_dt.resolution >= timedelta(microseconds=1)
+        return local_dt.replace(microsecond=utc_dt.microsecond)

tc_aws/loaders/s3_loader.py

@@ -5,9 +5,9 @@
 from tornado.concurrent import return_future
 import urllib2
 
-import tc_aws.connection
 import thumbor.loaders.http_loader as http_loader
 
+from tc_aws.aws.connection import get_connection
 
 def _get_bucket(url, root_path=None):
     """
@@ -57,7 +57,7 @@ def load(context, url, callback):
 
     if _validate_bucket(context, bucket):
         bucket_loader = Bucket(
-            connection=tc_aws.connection.get_connection(context),
+            connection=get_connection(context),
             name=bucket
         )
         file_key = None

tc_aws/result_storages/s3_storage.py

@@ -1,87 +1,22 @@
 #coding: utf-8
 
-import calendar
-from datetime import datetime, timedelta
-import hashlib
-import os
-
 from thumbor.result_storages import BaseStorage
-from thumbor.utils import logger
-
-from boto.s3.bucket import Bucket
-from boto.s3.key import Key
-from dateutil.parser import parse as parse_ts
-
-import tc_aws.connection
-
-class Storage(BaseStorage):
 
-    @property
-    def is_auto_webp(self):
-        return self.context.config.AUTO_WEBP and self.context.request.accepts_webp
+from ..aws.storage import AwsStorage
 
+class Storage(AwsStorage, BaseStorage):
     def __init__(self, context):
         BaseStorage.__init__(self, context)
-        self.storage = self.__get_s3_bucket()
-
-    def __get_s3_bucket(self):
-        return Bucket(
-            connection=tc_aws.connection.get_connection(self.context),
-            name=self.context.config.RESULT_STORAGE_BUCKET
-        )
+        AwsStorage.__init__(self, context, 'RESULT_STORAGE')
 
     def put(self, bytes):
-        file_abspath = self.normalize_path(self.context.request.url)
-        file_key=Key(self.storage)
-        file_key.key = file_abspath
-
-        file_key.set_contents_from_string(bytes,
-            encrypt_key = self.context.config.S3_STORAGE_SSE,
-            reduced_redundancy = self.context.config.S3_STORAGE_RRS
-        )
-
-        return file_abspath
-
-    def get(self):
-        file_abspath = self.normalize_path(self.context.request.url)
-        file_key = self.storage.get_key(file_abspath)
-
-        if not file_key or self.is_expired(file_key):
-            logger.debug("[RESULT_STORAGE] s3 key not found at %s" % file_abspath)
-            return None
-
-        return file_key.read()
-
-    def normalize_path(self, path):
-        root_path = self.context.config.RESULT_STORAGE_AWS_STORAGE_ROOT_PATH
-        path = path.lstrip('/')  # Remove leading '/'
-        path_segments = [path]
-        if self.is_auto_webp:
-            path_segments.append("webp")
-        return os.path.join(root_path, *path_segments)
-
-    def is_expired(self, key):
-        if key:
-            timediff = datetime.now() - self.utc_to_local(parse_ts(key.last_modified))
-            return timediff.seconds > self.context.config.STORAGE_EXPIRATION_SECONDS
-        else:
-            #If our key is bad just say we're expired
-            return True
-
-    def last_updated(self):
-        path = self.context.request.url
-        file_abspath = self.normalize_path(path)
-        file_key = self.storage.get_key(file_abspath)
+        path = self.normalize_path(self.context.request.url)
+        self.set(bytes, path)
 
-        if not file_key or self.is_expired(file_key):
-            logger.debug("[RESULT_STORAGE] s3 key not found at %s" % file_abspath)
-            return None
+        return path
 
-        return self.utc_to_local(parse_ts(file_key.last_modified))
+    def get(self, path=None):
+        if path == None:
+            path=self.normalize_path(self.context.request.url)
 
-    def utc_to_local(self,utc_dt):
-        # get integer timestamp to avoid precision lost
-        timestamp = calendar.timegm(utc_dt.timetuple())
-        local_dt = datetime.fromtimestamp(timestamp)
-        assert utc_dt.resolution >= timedelta(microseconds=1)
-        return local_dt.replace(microsecond=utc_dt.microsecond)
+        return super(Storage, self).get(path)

tc_aws/storages/s3_storage.py

@@ -1,65 +1,35 @@
 #coding: utf-8
 
-import calendar
-from datetime import datetime, timedelta
-import hashlib
 from json import loads, dumps
 
-from os.path import splitext, join
+from os.path import splitext
 
 from thumbor.storages import BaseStorage
-from thumbor.utils import logger
 
-from boto.s3.bucket import Bucket
-from boto.s3.key import Key
-from dateutil.parser import parse as parse_ts
-
-import tc_aws.connection
-
-class Storage(BaseStorage):
+from ..aws.storage import AwsStorage
 
+class Storage(AwsStorage, BaseStorage):
 
     def __init__(self, context):
         BaseStorage.__init__(self, context)
-        self.storage = self.__get_s3_bucket()
-
-    def __get_s3_bucket(self):
-        return Bucket(
-            connection=tc_aws.connection.get_connection(self.context),
-            name=self.context.config.STORAGE_BUCKET
-        )
+        AwsStorage.__init__(self, context, 'STORAGE')
 
     def put(self, path, bytes):
-        file_abspath = self.normalize_path(path)
-
-        file_key=Key(self.storage)
-        file_key.key = file_abspath
-
-        file_key.set_contents_from_string(bytes,
-            encrypt_key = self.context.config.S3_STORAGE_SSE,
-            reduced_redundancy = self.context.config.S3_STORAGE_RRS
-        )
+        self.set(bytes, self.normalize_path(path))
 
         return path
 
     def put_crypto(self, path):
         if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
             return
 
-        file_abspath = self.normalize_path(path)
-
         if not self.context.server.security_key:
             raise RuntimeError("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified")
 
+        file_abspath = self.normalize_path(path)
         crypto_path = '%s.txt' % splitext(file_abspath)[0]
 
-        file_key=Key(self.storage)
-        file_key.key = crypto_path
-
-        file_key.set_contents_from_string(self.context.server.security_key,
-            encrypt_key = self.context.config.S3_STORAGE_SSE,
-            reduced_redundancy = self.context.config.S3_STORAGE_RRS
-        )
+        self.set(self.context.server.security_key, crypto_path)
 
         return crypto_path
 
@@ -68,13 +38,7 @@ def put_detector_data(self, path, data):
 
         path = '%s.detectors.txt' % splitext(file_abspath)[0]
 
-        file_key=Key(self.storage)
-        file_key.key = path
-
-        file_key.set_contents_from_string(dumps(data),
-            encrypt_key = self.context.config.S3_STORAGE_SSE,
-            reduced_redundancy = self.context.config.S3_STORAGE_RRS
-        )
+        self.set(dumps(data), path)
 
         return path
 
@@ -83,19 +47,8 @@ def get_crypto(self, path):
         crypto_path = "%s.txt" % (splitext(file_abspath)[0])
 
         file_key = self.storage.get_key(crypto_path)
-        if not file_key:
-            return None
 
-        return file_key.read()
-
-    def get(self, path):
-
-        file_abspath = self.normalize_path(path)
-
-        file_key = self.storage.get_key(file_abspath)
-
-        if not file_key or self.is_expired(file_key):
-            logger.debug("[STORAGE] s3 key not found at %s" % file_abspath)
+        if not file_key:
             return None
 
         return file_key.read()
@@ -106,54 +59,28 @@ def get_detector_data(self, path):
 
         file_key = self.storage.get_key(path)
 
-        if not file_key or self.is_expired(path):
+        if not file_key or self.is_expired(file_key):
             return None
 
         return loads(file_key.read())
 
     def exists(self, path):
         file_abspath = self.normalize_path(path)
         file_key = self.storage.get_key(file_abspath)
+
         if not file_key:
             return False
-        return True
 
-    def normalize_path(self, path):
-        root_path = self.context.config.STORAGE_AWS_STORAGE_ROOT_PATH
-        path = path.lstrip('/')  # Remove leading '/'
-        path_segments = [root_path, path]
-        return join(*path_segments)
-
-
-    def is_expired(self, key):
-        if key:
-            expire_in_seconds = self.context.config.RESULT_STORAGE_EXPIRATION_SECONDS
-
-            #Never expire
-            if expire_in_seconds is None or expire_in_seconds == 0:
-                return False
-
-            timediff = datetime.now() - self.utc_to_local(parse_ts(key.last_modified))
-            return timediff.seconds > expire_in_seconds
-        else:
-            #If our key is bad just say we're expired
-            return True
+        return True
 
     def remove(self, path):
-
         if not self.exists(path):
             return
 
         if not self.storage.delete_key(path):
             return False
-        return True
 
-    def utc_to_local(self, utc_dt):
-        # get integer timestamp to avoid precision lost
-        timestamp = calendar.timegm(utc_dt.timetuple())
-        local_dt = datetime.fromtimestamp(timestamp)
-        assert utc_dt.resolution >= timedelta(microseconds=1)
-        return local_dt.replace(microsecond=utc_dt.microsecond)
+        return True
 
     def resolve_original_photo_path(self,filename):
         return filename