From 11af09e6f0dde1a9023803e81d529736174785e6 Mon Sep 17 00:00:00 2001 From: Nathan Cahill Date: Mon, 1 Sep 2014 16:33:05 -0600 Subject: [PATCH] multiple backends --- README.md | 118 +++++++++++++++++++++++--------- mimicdb/__init__.py | 26 ++++--- mimicdb/backends/__init__.py | 27 ++++++++ mimicdb/backends/default.py | 33 +++++++++ mimicdb/backends/memory.py | 41 +++++++++++ mimicdb/backends/sqlite.py | 78 +++++++++++++++++++++ mimicdb/{s3 => backends}/tpl.py | 0 mimicdb/s3/bucket.py | 40 +++++------ mimicdb/s3/connection.py | 38 +++++----- mimicdb/s3/key.py | 19 ++--- tests/__init__.py | 61 ++++++++--------- 11 files changed, 357 insertions(+), 124 deletions(-) create mode 100644 mimicdb/backends/__init__.py create mode 100644 mimicdb/backends/default.py create mode 100644 mimicdb/backends/memory.py create mode 100644 mimicdb/backends/sqlite.py rename mimicdb/{s3 => backends}/tpl.py (100%) diff --git a/README.md b/README.md index 4363f7e..704a8c2 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,105 @@ -### MimicDB: An Isomorphic Key-Value Store for S3 +## MimicDB -#### S3 Metadata without the Latency or Costs +[![PyPI](http://img.shields.io/pypi/v/mimicdb.svg?style=flat)](https://pypi.python.org/pypi/mimicdb/) +[![Build Status](http://img.shields.io/travis/nathancahill/mimicdb/master.svg?style=flat)](https://travis-ci.org/nathancahill/mimicdb) +[![Coverage Status](http://img.shields.io/coveralls/nathancahill/mimicdb/master.svg?style=flat)](https://coveralls.io/r/nathancahill/mimicdb) -MimicDB is a local database of the metadata of objects stored on S3. Many tasks like listing, searching keys and calculating storage usage can be completely handled locally, without the latency or costs of calling the S3 API. -On average, tasks like these are __2000x__ faster using MimicDB. +#### Python Implementation of MimicDB + +Python works with the Boto library. + +#### Installation + +By default, MimicDB requires Redis (although other backends can be used instead). -__Boto__ -```python ->>> c = S3Connection(KEY, SECRET) ->>> bucket = c.get_bucket('bucket_name') ->>> start = time.time() ->>> bucket.get_all_keys() ->>> print time.time() - start -0.425064992905 ``` +$ pip install redis +$ pip install mimicdb +``` + +#### Quickstart -__Boto + MimicDB__ -```python ->>> c = S3Connection(KEY, SECRET) ->>> bucket = c.get_bucket('bucket_name') ->>> start = time.time() ->>> bucket.get_all_keys() ->>> print time.time() - start -0.000198841094971 +If you're using Boto already, replace ```boto``` imports with ```mimicdb``` imports. + +Change: +``` +from boto.s3.connection import S3Connection +from boto.s3.key import Key ``` -#### Key Value Store +To: +``` +from mimicdb.s3.connection import S3Connection +from mimicdb.s3.key import Key +``` -MimicDB uses a Redis backend to stored S3 metadata. Data is stored in the following layout. +Additionally, import the MimicDB object itself, and initiate the backend: +``` +from mimicdb import MimicDB +MimicDB() +``` -`mimicdb` A set of buckets +After establishing a connection for the first time, sync the connection to save the metadata locally: +``` +conn = S3Connection(KEY, SECRET) +conn.sync() +``` -`mimicdb:bucket` A set of keys +Or sync only a couple buckets from the connection: +``` +conn.sync('bucket1', 'bucket2') +``` -`mimicdb:bucket:key` A hash of key metadata (size and MD5) +After that, upload, download and list as you usually would. API calls that can be responded to locally will return instantly without hitting S3 servers. API calls that are made to S3 using MimicDB will be mimicked locally to ensure consistency with the remote servers. -The `mimicdb` prefix can additionally use an optional `namespace` string, which allows multiple S3 connections to share the same backend. In that case, the layout looks like this: +Pass ```force=True``` to most functions to force a call to the S3 API. This also updates the local database. -`mimicdb:namespace` +#### Alternate Backends -`mimicdb:namespace:bucket` +Besides the default Redis backend, MimicDB has SQLite and in-memory backends available. +``` +from mimicdb.backends.sqlite import SQLite +MimicDB(SQLite()) +``` +``` +from mimicdb.backends.memory import Memory +MimicDB(Memory()) +``` + +#### Documentation + +[mimicdb.readthedocs.org](http://mimicdb.readthedocs.org) + +#### Contributing + + +1. Fork the repo. +2. Run tests to ensure a clean, working slate. +3. Improve/fix the code. +4. Add test cases if new functionality introduced or bug fixed (100% test coverage). +5. Ensure tests pass. +6. Push to your fork and submit a pull request to the develop branch. + +#### Tests -`mimicdb:namespace:bucket:key` +Run tests after installing nose and coverage. -#### Implementation +``` +$ nosetests --with-coverage --cover-package=mimicdb +``` + +Integration testing is provided by Travis-CI at [travis-ci.org/nathancahill/mimicdb](https://travis-ci.org/nathancahill/mimicdb) + +Test coverage reporting is provided by Coveralls at [coveralls.io/r/nathancahill/mimicdb](coveralls.io/r/nathancahill/mimicdb) -MimicDB is currently implemented in Python via Boto. If you're using Boto already, the MimicDB Python library works as a drop in replacement. +#### Benchmarks + +Run ```benchmarks.py``` in the root of the repo: + +``` +$ python benchmarks.py +Boto Time: 0.338411092758 +MimicDB Time: 0.00015789039612 +Factor: 2143x faster +``` diff --git a/mimicdb/__init__.py b/mimicdb/__init__.py index db18fcf..412b5e0 100644 --- a/mimicdb/__init__.py +++ b/mimicdb/__init__.py @@ -1,19 +1,17 @@ -from redis import StrictRedis -from .s3 import tpl +"""Python implementation of MimicDB +""" + class MimicDB(object): - def __init__(self, *args, **kwargs): + def __init__(self, backend=None, namespace=None): + """Initialze the MimicDB backend with an optional namespace. """ - Initialze the MimicDB object by passing the Redis connection parameters: - - :host='localhost' - :port=6379 - :db=0 + if not backend: + from .backends.default import Redis + backend = Redis() - The Redis connection is accessed elsewhere in the module by importing - mimicdb, then calling mimicdb.redis - """ - if kwargs and 'namespace' in kwargs: - tpl.set_namespace(kwargs.pop('namespace')) + globals()['backend'] = backend - globals()['redis'] = StrictRedis(*args, **kwargs) + if namespace: + from .backends import tpl + tpl.set_namespace(namespace) diff --git a/mimicdb/backends/__init__.py b/mimicdb/backends/__init__.py new file mode 100644 index 0000000..6d59426 --- /dev/null +++ b/mimicdb/backends/__init__.py @@ -0,0 +1,27 @@ +"""Base class for MimicDB backends +""" + +class Backend(object): + def __init__(self, *args, **kwargs): + pass + + def delete(self, *names): + pass + + def sadd(self, name, *values): + pass + + def srem(self, name, *values): + pass + + def sismember(self, name, value): + pass + + def smembers(self, name): + pass + + def hmset(self, name, mapping): + pass + + def hgetall(self, name): + pass diff --git a/mimicdb/backends/default.py b/mimicdb/backends/default.py new file mode 100644 index 0000000..6753c9f --- /dev/null +++ b/mimicdb/backends/default.py @@ -0,0 +1,33 @@ + +from redis import StrictRedis + +from . import Backend + + +class Redis(Backend): + def __init__(self, *args, **kwargs): + self._redis = StrictRedis(*args, **kwargs) + + def keys(self, pattern='*'): + return self._redis.keys(pattern) + + def delete(self, *names): + return self._redis.delete(*names) + + def sadd(self, name, *values): + return self._redis.sadd(name, *values) + + def srem(self, name, *values): + return self._redis.srem(name, *values) + + def sismember(self, name, value): + return self._redis.sismember(name, value) + + def smembers(self, name): + return self._redis.smembers(name) + + def hmset(self, name, mapping): + return self._redis.hmset(name, mapping) + + def hgetall(self, name): + return self._redis.hgetall(name) diff --git a/mimicdb/backends/memory.py b/mimicdb/backends/memory.py new file mode 100644 index 0000000..46802ce --- /dev/null +++ b/mimicdb/backends/memory.py @@ -0,0 +1,41 @@ + + +from . import Backend + + +class Memory(Backend): + def __init__(self): + self._data = dict() + + def keys(self, pattern='*'): + pattern = pattern.replace('*', '') + return [key for key in self._data if key.startswith(pattern)] + + def delete(self, *names): + for name in names: + self._data.pop(name, None) + + def sadd(self, name, *values): + if name in self._data: + self._data[name].update(values) + else: + self._data[name] = set(values) + + def srem(self, name, *values): + if name in self._data: + self._data[name].difference_update(values) + + def sismember(self, name, value): + if name in self._data: + return value in self._data[name] + + return False + + def smembers(self, name): + return self._data.get(name, []) + + def hmset(self, name, mapping): + self._data[name] = mapping + + def hgetall(self, name): + return self._data.get(name, dict()) diff --git a/mimicdb/backends/sqlite.py b/mimicdb/backends/sqlite.py new file mode 100644 index 0000000..84f4ea5 --- /dev/null +++ b/mimicdb/backends/sqlite.py @@ -0,0 +1,78 @@ + +import sqlite3 + +from . import Backend + + +class SQLite(Backend): + def __init__(self, *args, **kwargs): + if not args and not kwargs: + args = [':memory:'] + + self._sqlite = sqlite3.connect(*args, **kwargs) + + def keys(self, pattern='*'): + c = self._sqlite.cursor() + pattern = pattern.replace('*', '%') + c.execute('SELECT name FROM sqlite_master WHERE type="%s" AND name LIKE "%s"' % ('table', pattern)) + + return [row[0] for row in c.fetchall()] + + def delete(self, *names): + c = self._sqlite.cursor() + + for name in names: + c.execute('DROP TABLE IF EXISTS "%s"' % (name,)) + + self._sqlite.commit() + + def sadd(self, name, *values): + c = self._sqlite.cursor() + c.execute('CREATE TABLE IF NOT EXISTS "%s" (member text)' % (name,)) + + for value in values: + c.execute('INSERT INTO "%s" VALUES ("%s")' % (name, value)) + + self._sqlite.commit() + + def srem(self, name, *values): + c = self._sqlite.cursor() + c.execute('CREATE TABLE IF NOT EXISTS "%s" (member text)' % (name,)) + + for value in values: + c.execute('DELETE FROM "%s" WHERE member="%s"' % (name, value)) + + self._sqlite.commit() + + def sismember(self, name, value): + c = self._sqlite.cursor() + c.execute('CREATE TABLE IF NOT EXISTS "%s" (member text)' % (name,)) + c.execute('SELECT * FROM "%s" WHERE member="%s"' % (name, value)) + + return c.fetchone() != None + + def smembers(self, name): + c = self._sqlite.cursor() + c.execute('CREATE TABLE IF NOT EXISTS "%s" (member text)' % (name,)) + c.execute('SELECT * FROM "%s"' % (name,)) + + return [row[0] for row in c.fetchall()] + + def hmset(self, name, mapping): + c = self._sqlite.cursor() + c.execute('CREATE TABLE IF NOT EXISTS "%s" (size text, md5 text)' % (name,)) + c.execute('INSERT INTO "%s" VALUES ("%s", "%s")' % (name, mapping['size'], mapping['md5'])) + + self._sqlite.commit() + + def hgetall(self, name): + c = self._sqlite.cursor() + c.execute('CREATE TABLE IF NOT EXISTS "%s" (size text, md5 text)' % (name,)) + c.execute('SELECT * FROM "%s"' % (name,)) + + row = c.fetchone() + + if row: + return dict(size=row[0], md5=row[1]) + + return dict() diff --git a/mimicdb/s3/tpl.py b/mimicdb/backends/tpl.py similarity index 100% rename from mimicdb/s3/tpl.py rename to mimicdb/backends/tpl.py diff --git a/mimicdb/s3/bucket.py b/mimicdb/s3/bucket.py index 3889823..81f5c60 100644 --- a/mimicdb/s3/bucket.py +++ b/mimicdb/s3/bucket.py @@ -6,7 +6,7 @@ import mimicdb from .key import Key -from . import tpl +from ..backends import tpl class Bucket(BotoBucket): @@ -44,15 +44,15 @@ def _get_key_internal(self, *args, **kwargs): key, res = super(Bucket, self)._get_key_internal(*args, **kwargs) if key: - mimicdb.redis.sadd(tpl.bucket % self.name, key.name) - mimicdb.redis.hmset(tpl.key % (self.name, key.name), + mimicdb.backend.sadd(tpl.bucket % self.name, key.name) + mimicdb.backend.hmset(tpl.key % (self.name, key.name), dict(size=key.size, md5=key.etag.strip('"'))) return key, res key = None - if mimicdb.redis.sismember(tpl.bucket % self.name, args[0]): + if mimicdb.backend.sismember(tpl.bucket % self.name, args[0]): key = Key(self) key.name = args[0] @@ -80,19 +80,19 @@ def delete_keys(self, *args, **kwargs): break if isinstance(key, basestring): - mimicdb.redis.srem(tpl.bucket % self.name, key) - mimicdb.redis.delete(tpl.key % (self.name, key)) + mimicdb.backend.srem(tpl.bucket % self.name, key) + mimicdb.backend.delete(tpl.key % (self.name, key)) elif isinstance(key, BotoKey) or isinstance(key, Key): - mimicdb.redis.srem(tpl.bucket % self.name, key.name) - mimicdb.redis.delete(tpl.key % (self.name, key.name)) + mimicdb.backend.srem(tpl.bucket % self.name, key.name) + mimicdb.backend.delete(tpl.key % (self.name, key.name)) return super(Bucket, self).delete_keys(*args, **kwargs) def _delete_key_internal(self, *args, **kwargs): """Remove key name from bucket set. """ - mimicdb.redis.srem(tpl.bucket % self.name, args[0]) - mimicdb.redis.delete(tpl.key % (self.name, args[0])) + mimicdb.backend.srem(tpl.bucket % self.name, args[0]) + mimicdb.backend.delete(tpl.key % (self.name, args[0])) return super(Bucket, self)._delete_key_internal(*args, **kwargs) @@ -111,11 +111,11 @@ def list(self, *args, **kwargs): else: prefix = kwargs.get('prefix', args[0] if args else '') - for key in mimicdb.redis.smembers(tpl.bucket % self.name): + for key in mimicdb.backend.smembers(tpl.bucket % self.name): if key.startswith(prefix): k = Key(self, key) - meta = mimicdb.redis.hgetall(tpl.key % (self.name, key)) + meta = mimicdb.backend.hgetall(tpl.key % (self.name, key)) if meta: k._load_meta(meta['size'], meta['md5']) @@ -138,8 +138,8 @@ def _get_all(self, *args, **kwargs): keys = super(Bucket, self)._get_all(*args, **kwargs) for key in keys: - mimicdb.redis.sadd(tpl.bucket % self.name, key.name) - mimicdb.redis.hmset(tpl.key % (self.name, key.name), dict(size=key.size, md5=key.etag.strip('"'))) + mimicdb.backend.sadd(tpl.bucket % self.name, key.name) + mimicdb.backend.hmset(tpl.key % (self.name, key.name), dict(size=key.size, md5=key.etag.strip('"'))) key.name = key.name @@ -152,12 +152,12 @@ def _get_all(self, *args, **kwargs): def sync(self): """Sync the bucket set with S3. """ - for key in mimicdb.redis.smembers(tpl.bucket % self.name): - mimicdb.redis.delete(tpl.key % (self.name, key)) + for key in mimicdb.backend.smembers(tpl.bucket % self.name): + mimicdb.backend.delete(tpl.key % (self.name, key)) - mimicdb.redis.delete(tpl.bucket % self.name) - mimicdb.redis.sadd(tpl.connection, self.name) + mimicdb.backend.delete(tpl.bucket % self.name) + mimicdb.backend.sadd(tpl.connection, self.name) for key in self.list(force=True): - mimicdb.redis.sadd(tpl.bucket % self.name, key.name) - mimicdb.redis.hmset(tpl.key % (self.name, key.name), dict(size=key.size, md5=key.etag.strip('"'))) + mimicdb.backend.sadd(tpl.bucket % self.name, key.name) + mimicdb.backend.hmset(tpl.key % (self.name, key.name), dict(size=key.size, md5=key.etag.strip('"'))) diff --git a/mimicdb/s3/connection.py b/mimicdb/s3/connection.py index d7d9391..568876c 100644 --- a/mimicdb/s3/connection.py +++ b/mimicdb/s3/connection.py @@ -6,7 +6,7 @@ import mimicdb from .bucket import Bucket -from . import tpl +from ..backends import tpl class S3Connection(BotoS3Connection): @@ -25,11 +25,11 @@ def get_all_buckets(self, *args, **kwargs): buckets = super(S3Connection, self).get_all_buckets(*args, **kwargs) for bucket in buckets: - mimicdb.redis.sadd(tpl.connection, bucket.name) + mimicdb.backend.sadd(tpl.connection, bucket.name) return buckets - return [Bucket(self, bucket) for bucket in mimicdb.redis.smembers(tpl.connection)] + return [Bucket(self, bucket) for bucket in mimicdb.backend.smembers(tpl.connection)] def get_bucket(self, bucket_name, validate=True, headers=None, force=None): """Return a bucket from the MimicDB set if it exists. Simulates an @@ -37,10 +37,10 @@ def get_bucket(self, bucket_name, validate=True, headers=None, force=None): """ if force: bucket = super(S3Connection, self).get_bucket(bucket_name, validate, headers) - mimicdb.redis.sadd(tpl.connection, bucket.name) + mimicdb.backend.sadd(tpl.connection, bucket.name) return bucket - if mimicdb.redis.sismember(tpl.connection, bucket_name): + if mimicdb.backend.sismember(tpl.connection, bucket_name): return Bucket(self, bucket_name) else: if validate: @@ -54,7 +54,7 @@ def create_bucket(self, *args, **kwargs): bucket = super(S3Connection, self).create_bucket(*args, **kwargs) if bucket: - mimicdb.redis.sadd(tpl.connection, bucket.name) + mimicdb.backend.sadd(tpl.connection, bucket.name) return bucket @@ -68,9 +68,9 @@ def delete_bucket(self, *args, **kwargs): bucket = kwargs.get('bucket_name', args[0] if args else None) if bucket: - mimicdb.redis.srem(tpl.connection, bucket) + mimicdb.backend.srem(tpl.connection, bucket) - def sync(self, buckets=[]): + def sync(self, *buckets): """Sync either a list of buckets or the entire connection. Force all API calls to S3 and populate the database with the current @@ -78,24 +78,24 @@ def sync(self, buckets=[]): """ if buckets: for _bucket in buckets: - for key in mimicdb.redis.smembers(tpl.bucket % _bucket): - mimicdb.redis.delete(tpl.key % (_bucket, key)) + for key in mimicdb.backend.smembers(tpl.bucket % _bucket): + mimicdb.backend.delete(tpl.key % (_bucket, key)) - mimicdb.redis.delete(tpl.bucket % _bucket) + mimicdb.backend.delete(tpl.bucket % _bucket) bucket = self.get_bucket(_bucket, force=True) for key in bucket.list(force=True): - mimicdb.redis.sadd(tpl.bucket % bucket, key.name) - mimicdb.redis.hmset(tpl.key % (bucket, key.name), dict(size=key.size, md5=key.etag.strip('"'))) + mimicdb.backend.sadd(tpl.bucket % bucket, key.name) + mimicdb.backend.hmset(tpl.key % (bucket, key.name), dict(size=key.size, md5=key.etag.strip('"'))) else: - for bucket in mimicdb.redis.smembers(tpl.connection): - for key in mimicdb.redis.smembers(tpl.bucket % bucket): - mimicdb.redis.delete(tpl.key % (bucket, key)) + for bucket in mimicdb.backend.smembers(tpl.connection): + for key in mimicdb.backend.smembers(tpl.bucket % bucket): + mimicdb.backend.delete(tpl.key % (bucket, key)) - mimicdb.redis.delete(tpl.bucket % bucket) + mimicdb.backend.delete(tpl.bucket % bucket) for bucket in self.get_all_buckets(force=True): for key in bucket.list(force=True): - mimicdb.redis.sadd(tpl.bucket % bucket.name, key.name) - mimicdb.redis.hmset(tpl.key % (bucket.name, key.name), dict(size=key.size, md5=key.etag.strip('"'))) + mimicdb.backend.sadd(tpl.bucket % bucket.name, key.name) + mimicdb.backend.hmset(tpl.key % (bucket.name, key.name), dict(size=key.size, md5=key.etag.strip('"'))) diff --git a/mimicdb/s3/key.py b/mimicdb/s3/key.py index e74a9ae..cadb13d 100644 --- a/mimicdb/s3/key.py +++ b/mimicdb/s3/key.py @@ -4,7 +4,8 @@ from boto.s3.key import Key as BotoKey import mimicdb -from . import tpl + +from ..backends import tpl class Key(BotoKey): @@ -18,10 +19,10 @@ def __init__(self, *args, **kwargs): self._name = name if name and bucket: - meta = mimicdb.redis.hgetall(tpl.key % (bucket.name, name)) + meta = mimicdb.backend.hgetall(tpl.key % (bucket.name, name)) if meta: - mimicdb.redis.sadd(tpl.bucket % bucket.name, name) + mimicdb.backend.sadd(tpl.bucket % bucket.name, name) self._load_meta(meta['size'], meta['md5']) super(Key, self).__init__(*args, **kwargs) @@ -52,10 +53,10 @@ def name(self, value): self._name = value if value: - meta = mimicdb.redis.hgetall(tpl.key % (self.bucket.name, value)) + meta = mimicdb.backend.hgetall(tpl.key % (self.bucket.name, value)) if meta: - mimicdb.redis.sadd(tpl.bucket % self.bucket.name, value) + mimicdb.backend.sadd(tpl.bucket % self.bucket.name, value) self._load_meta(meta['size'], meta['md5']) def _send_file_internal(self, *args, **kwargs): @@ -64,8 +65,8 @@ def _send_file_internal(self, *args, **kwargs): """ super(Key, self)._send_file_internal(*args, **kwargs) - mimicdb.redis.sadd(tpl.bucket % self.bucket.name, self.name) - mimicdb.redis.hmset(tpl.key % (self.bucket.name, self.name), + mimicdb.backend.sadd(tpl.bucket % self.bucket.name, self.name) + mimicdb.backend.hmset(tpl.key % (self.bucket.name, self.name), dict(size=self.size, md5=self.md5)) def _get_file_internal(self, *args, **kwargs): @@ -74,6 +75,6 @@ def _get_file_internal(self, *args, **kwargs): """ super(Key, self)._get_file_internal(*args, **kwargs) - mimicdb.redis.sadd(tpl.bucket % self.bucket.name, self.name) - mimicdb.redis.hmset(tpl.key % (self.bucket.name, self.name), + mimicdb.backend.sadd(tpl.bucket % self.bucket.name, self.name) + mimicdb.backend.hmset(tpl.key % (self.bucket.name, self.name), dict(size=self.size, md5=self.md5)) diff --git a/tests/__init__.py b/tests/__init__.py index a16043a..7bc5f12 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,8 +5,6 @@ import logging import unittest -import redis - from boto.s3.connection import S3Connection as BotoS3Connection from boto.s3.bucket import Bucket as BotoBucket from boto.s3.key import Key as BotoKey @@ -17,7 +15,8 @@ from mimicdb.s3.connection import S3Connection from mimicdb.s3.bucket import Bucket from mimicdb.s3.key import Key -from mimicdb.s3 import tpl +from mimicdb.backends import tpl +from mimicdb.backends.sqlite import SQLite AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] @@ -30,12 +29,12 @@ def random_string(size=6, chars=string.ascii_lowercase + string.digits): class MimicDBTestCase(unittest.TestCase): def setUp(self): - MimicDB(namespace='tests') + MimicDB(SQLite()) + # MimicDB(namespace='tests') self.conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) self.boto_conn = BotoS3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) - self.redis = redis.StrictRedis() self.name = 'mimicdb-tests-' + random_string() self.boto_bucket = self.boto_conn.create_bucket(self.name) @@ -51,11 +50,11 @@ def tearDown(self): bucket.delete() - for key in self.redis.keys(tpl.connection + '*'): - self.redis.delete(key) + for key in mimicdb.backend.keys(tpl.connection + '*'): + mimicdb.backend.delete(key) - def redis_test(self): - self.assertIsInstance(mimicdb.redis, redis.StrictRedis) + def backend_test(self): + self.assertIsInstance(mimicdb.backend, mimicdb.backends.Backend) def get_all_buckets_test(self): buckets = self.conn.get_all_buckets() @@ -77,20 +76,20 @@ def get_no_validate_bucket_test(self): def get_bucket_force_test(self): self.conn.get_bucket(self.name, force=True) - assert self.name in self.redis.smembers(tpl.connection) + assert self.name in mimicdb.backend.smembers(tpl.connection) def create_bucket_test(self): bucket = self.conn.create_bucket(self.name + '-create') - assert self.name + '-create' in self.redis.smembers(tpl.connection) + assert self.name + '-create' in mimicdb.backend.smembers(tpl.connection) def sync_test(self): self.conn.sync() - assert self.name in self.redis.smembers(tpl.connection) - assert 'upload' in self.redis.smembers(tpl.bucket % self.name) + assert self.name in mimicdb.backend.smembers(tpl.connection) + assert 'upload' in mimicdb.backend.smembers(tpl.bucket % self.name) - meta = self.redis.hgetall(tpl.key % (self.name, 'upload')) + meta = mimicdb.backend.hgetall(tpl.key % (self.name, 'upload')) assert int(meta['size']) == len('upload') assert meta['md5'] == hashlib.md5('upload').hexdigest() @@ -101,7 +100,7 @@ def delete_bucket_not_empty_test(self): with self.assertRaises(S3ResponseError): self.conn.delete_bucket(self.name) - assert self.name in self.redis.smembers(tpl.connection) + assert self.name in mimicdb.backend.smembers(tpl.connection) def delete_bucket_empty_test(self): for key in self.boto_bucket.list(): @@ -110,15 +109,15 @@ def delete_bucket_empty_test(self): self.conn.sync() self.conn.delete_bucket(self.name) - assert self.name not in self.redis.smembers(tpl.connection) + assert self.name not in mimicdb.backend.smembers(tpl.connection) def sync_bucket_test(self): self.boto_conn.create_bucket(self.name + '-sync') - self.conn.sync(buckets=[self.name]) + self.conn.sync(self.name) - assert self.name in self.redis.smembers(tpl.connection) - assert self.name + '-sync' not in self.redis.smembers(tpl.connection) + assert self.name in mimicdb.backend.smembers(tpl.connection) + assert self.name + '-sync' not in mimicdb.backend.smembers(tpl.connection) def sync_clear_test(self): self.conn.sync() @@ -129,7 +128,7 @@ def sync_clear_test(self): self.conn.sync() - meta = self.redis.hgetall(tpl.key % (self.name, 'upload')) + meta = mimicdb.backend.hgetall(tpl.key % (self.name, 'upload')) assert int(meta['size']) == len('sync_clear') assert meta['md5'] == hashlib.md5('sync_clear').hexdigest() @@ -141,9 +140,9 @@ def sync_bucket_clear_test(self): key.key = 'upload' key.set_contents_from_string('sync_clear') - self.conn.sync(buckets=[self.name]) + self.conn.sync(self.name) - meta = self.redis.hgetall(tpl.key % (self.name, 'upload')) + meta = mimicdb.backend.hgetall(tpl.key % (self.name, 'upload')) assert int(meta['size']) == len('sync_clear') assert meta['md5'] == hashlib.md5('sync_clear').hexdigest() @@ -182,7 +181,7 @@ def delete_keys_string_test(self): key = boto_bucket.get_key('upload') assert key is None - assert 'upload' not in self.redis.smembers(tpl.bucket % self.name) + assert 'upload' not in mimicdb.backend.smembers(tpl.bucket % self.name) def delete_keys_key_test(self): self.conn.sync() @@ -195,7 +194,7 @@ def delete_keys_key_test(self): key = boto_bucket.get_key('upload') assert key is None - assert 'upload' not in self.redis.smembers(tpl.bucket % self.name) + assert 'upload' not in mimicdb.backend.smembers(tpl.bucket % self.name) def delete_key_test(self): self.conn.sync() @@ -206,7 +205,7 @@ def delete_key_test(self): key = self.boto_bucket.get_key('upload') assert key is None - assert 'upload' not in self.redis.smembers(tpl.bucket % self.name) + assert 'upload' not in mimicdb.backend.smembers(tpl.bucket % self.name) def iter_test(self): self.conn.sync() @@ -312,7 +311,7 @@ def track_upload_test(self): key.key = 'track' key.set_contents_from_string('track') - assert 'track' in self.redis.smembers(tpl.bucket % self.name) + assert 'track' in mimicdb.backend.smembers(tpl.bucket % self.name) def track_upload_init_test(self): self.conn.sync() @@ -322,7 +321,7 @@ def track_upload_init_test(self): key = Key(bucket, 'track') key.set_contents_from_string('track') - assert 'track' in self.redis.smembers(tpl.bucket % self.name) + assert 'track' in mimicdb.backend.smembers(tpl.bucket % self.name) def track_upload_meta_test(self): self.conn.sync() @@ -333,7 +332,7 @@ def track_upload_meta_test(self): key.key = 'track' key.set_contents_from_string('track') - meta = self.redis.hgetall(tpl.key % (self.name, 'track')) + meta = mimicdb.backend.hgetall(tpl.key % (self.name, 'track')) assert int(meta['size']) == len('track') assert meta['md5'] == hashlib.md5('track').hexdigest() @@ -346,7 +345,7 @@ def track_upload_meta_init_test(self): key = Key(bucket, 'track') key.set_contents_from_string('track') - meta = self.redis.hgetall(tpl.key % (self.name, 'track')) + meta = mimicdb.backend.hgetall(tpl.key % (self.name, 'track')) assert int(meta['size']) == len('track') assert meta['md5'] == hashlib.md5('track').hexdigest() @@ -364,7 +363,7 @@ def track_download_meta_test(self): key.key = 'download' key.get_contents_as_string() - meta = self.redis.hgetall(tpl.key % (self.name, 'download')) + meta = mimicdb.backend.hgetall(tpl.key % (self.name, 'download')) assert int(meta['size']) == len('download') assert meta['md5'] == hashlib.md5('download').hexdigest() @@ -381,7 +380,7 @@ def track_download_meta_init_test(self): key = Key(bucket, 'download') key.get_contents_as_string() - meta = self.redis.hgetall(tpl.key % (self.name, 'download')) + meta = mimicdb.backend.hgetall(tpl.key % (self.name, 'download')) assert int(meta['size']) == len('download') assert meta['md5'] == hashlib.md5('download').hexdigest()