From 4794a7e36c74f14a236a69616e257b8a756dc118 Mon Sep 17 00:00:00 2001 From: "Valentin v. Seggern" Date: Tue, 11 Aug 2015 20:51:57 +0200 Subject: [PATCH 1/3] added dynamodb backend for AWS deployments without redis --- sorl/thumbnail/kvstores/dynamodb_kvstore.py | 37 +++++++++++++++++++++ tests/settings/dynamodb.py | 8 +++++ tox.ini | 4 ++- 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 sorl/thumbnail/kvstores/dynamodb_kvstore.py create mode 100644 tests/settings/dynamodb.py diff --git a/sorl/thumbnail/kvstores/dynamodb_kvstore.py b/sorl/thumbnail/kvstores/dynamodb_kvstore.py new file mode 100644 index 000000000..d5e2107aa --- /dev/null +++ b/sorl/thumbnail/kvstores/dynamodb_kvstore.py @@ -0,0 +1,37 @@ +from __future__ import unicode_literals + +from boto.dynamodb2.table import Table +import boto +import json +from sorl.thumbnail.kvstores.base import KVStoreBase +from sorl.thumbnail.conf import settings + + +class KVStore(KVStoreBase): + def __init__(self): + super(KVStore, self).__init__() + conn = boto.dynamodb2.connect_to_region(settings.AWS_REGION_NAME, + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY) + self.table = Table(settings.THUMBNAIL_DYNAMODB_NAME, connection=conn) + + def _get_raw(self, key): + try: + return self.table.get_item(key=key)['value'] + except boto.dynamodb2.exceptions.ItemNotFound: + pass + + def _set_raw(self, key, value): + try: + item = self.table.get_item(key=key) + except boto.dynamodb2.exceptions.ItemNotFound: + item = self.table.new_item() + item['key'] = key + item['value'] = value + item.save() + + def _delete_raw(self, *keys): + [self.table.delete_item(key=k) for k in keys] + + def _find_keys_raw(self, prefix): + return [i['key'] for i in self.table.scan(key__beginswith=prefix)] diff --git a/tests/settings/dynamodb.py b/tests/settings/dynamodb.py new file mode 100644 index 000000000..17afdee34 --- /dev/null +++ b/tests/settings/dynamodb.py @@ -0,0 +1,8 @@ +from .default import * + + +THUMBNAIL_KVSTORE = 'sorl.thumbnail.kvstores.dynamodb_kvstore.KVStore' +THUMBNAIL_DYNAMODB_NAME = 'test' +AWS_REGION_NAME = 'eu-central-1' +AWS_ACCESS_KEY_ID = 'use' +AWS_SECRET_ACCESS_KEY = 'yours' diff --git a/tox.ini b/tox.ini index 3d3aa2067..011bd92e2 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ django_find_project = false skipsdist = True envlist = {py27}-django14-{pil,imagemagick,graphicsmagick,redis,wand,pgmagick,dbm}, - {py27,py34}-django{15,16,17,18}-{pil,imagemagick,graphicsmagick,redis,wand,pgmagick,dbm} + {py27,py34}-django{15,16,17,18}-{pil,imagemagick,graphicsmagick,redis,dynamodb,wand,pgmagick,dbm} [testenv] changedir = {toxinidir}/tests @@ -20,6 +20,7 @@ deps = pytest-django Pillow redis: redis + dynamodb: boto pgmagick: pgmagick wand: wand django14: Django>=1.4,<1.5 @@ -34,6 +35,7 @@ setenv = imagemagick: DJANGO_SETTINGS_MODULE=tests.settings.imagemagick graphicsmagick: DJANGO_SETTINGS_MODULE=tests.settings.graphicsmagick redis: DJANGO_SETTINGS_MODULE=tests.settings.redis + dynamodb: DJANGO_SETTINGS_MODULE=tests.settings.dynamodb wand: DJANGO_SETTINGS_MODULE=tests.settings.wand pgmagick: DJANGO_SETTINGS_MODULE=tests.settings.pgmagick dbm: DJANGO_SETTINGS_MODULE=tests.settings.dbm From 3c718386eb594db5fa49c11e4bbc1cd52b4611ed Mon Sep 17 00:00:00 2001 From: "Valentin v. Seggern" Date: Tue, 25 Aug 2015 12:43:33 +0200 Subject: [PATCH 2/3] overwriting on save() to make sure there are no problems with the expects() call --- sorl/thumbnail/kvstores/dynamodb_kvstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorl/thumbnail/kvstores/dynamodb_kvstore.py b/sorl/thumbnail/kvstores/dynamodb_kvstore.py index d5e2107aa..7bf525a7f 100644 --- a/sorl/thumbnail/kvstores/dynamodb_kvstore.py +++ b/sorl/thumbnail/kvstores/dynamodb_kvstore.py @@ -28,7 +28,7 @@ def _set_raw(self, key, value): item = self.table.new_item() item['key'] = key item['value'] = value - item.save() + item.save(overwrite=True) def _delete_raw(self, *keys): [self.table.delete_item(key=k) for k in keys] From e9983a388ba250d0e091068ed5c5f95cc860fea6 Mon Sep 17 00:00:00 2001 From: "Valentin v. Seggern" Date: Thu, 29 Oct 2015 12:34:42 +0100 Subject: [PATCH 3/3] codestyle -> pep8 --- sorl/thumbnail/kvstores/dynamodb_kvstore.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sorl/thumbnail/kvstores/dynamodb_kvstore.py b/sorl/thumbnail/kvstores/dynamodb_kvstore.py index 7bf525a7f..013f21c0b 100644 --- a/sorl/thumbnail/kvstores/dynamodb_kvstore.py +++ b/sorl/thumbnail/kvstores/dynamodb_kvstore.py @@ -2,7 +2,6 @@ from boto.dynamodb2.table import Table import boto -import json from sorl.thumbnail.kvstores.base import KVStoreBase from sorl.thumbnail.conf import settings @@ -10,9 +9,11 @@ class KVStore(KVStoreBase): def __init__(self): super(KVStore, self).__init__() - conn = boto.dynamodb2.connect_to_region(settings.AWS_REGION_NAME, - aws_access_key_id=settings.AWS_ACCESS_KEY_ID, - aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY) + region = settings.AWS_REGION_NAME + access_key = settings.AWS_ACCESS_KEY_ID + secret = settings.AWS_SECRET_ACCESS_KEY + conn = boto.dynamodb2.connect_to_region(region, aws_access_key_id=access_key, + aws_secret_access_key=secret) self.table = Table(settings.THUMBNAIL_DYNAMODB_NAME, connection=conn) def _get_raw(self, key):