Permalink
Browse files

Bug fix: Engine namespace is truly engine-local

  • Loading branch information...
stevearc committed Mar 12, 2014
1 parent bf3261c commit 3b4fad748c35fa16d853896e5f56559c25ecdf28
Showing with 98 additions and 57 deletions.
  1. +1 −0 .gitignore
  2. +25 −17 flywheel/engine.py
  3. +35 −20 flywheel/model_meta.py
  4. +1 −1 flywheel/query.py
  5. +10 −5 tests/test_fields.py
  6. +26 −14 tests/test_models.py
@@ -37,6 +37,7 @@ nosetests.xml
flywheel_env
.ropeproject
_localhost.db
_local.db
# docs
_build
@@ -20,8 +20,9 @@ class Engine(object):
Parameters
----------
dynamo : :class:`dynamodb3.DynamoDBConnection`, optional
namespace : list, optional
List of namespace component strings for models
namespace : list or str, optional
String prefix or list of component parts of a prefix for models. All
table names will be prefixed by this string or strings (joined by '-').
default_conflict : {'update', 'overwrite', 'raise'}, optional
Default setting for delete(), save(), and sync() (default 'update')
@@ -72,11 +73,10 @@ class Engine(object):
"""
def __init__(self, dynamo=None, namespace=None, default_conflict='update'):
def __init__(self, dynamo=None, namespace=(), default_conflict='update'):
self.dynamo = dynamo
self.models = {}
self.namespace = namespace or []
ModelMetadata.namespace = self.namespace
self.namespace = namespace
self._default_conflict = None
self.default_conflict = default_conflict
@@ -135,6 +135,10 @@ def connect_to_region(self, region, **kwargs):
""" Connect to an AWS region """
self.dynamo = DynamoDBConnection.connect_to_region(region, **kwargs)
def connect_to_host(self, **kwargs):
""" Connect to a specific host """
self.dynamo = DynamoDBConnection.connect_to_host(**kwargs)
def register(self, *models):
"""
Register one or more models with the engine
@@ -154,7 +158,8 @@ def create_schema(self, test=False):
changed = []
for model in six.itervalues(self.models):
result = model.meta_.create_dynamo_schema(self.dynamo, tablenames,
test=test, wait=True)
test=test, wait=True,
namespace=self.namespace)
if result:
changed.append(result)
return changed
@@ -165,15 +170,16 @@ def delete_schema(self, test=False):
changed = []
for model in six.itervalues(self.models):
result = model.meta_.delete_dynamo_schema(self.dynamo, tablenames,
test=test, wait=True)
test=test, wait=True,
namespace=self.namespace)
changed.append(result)
return changed
def get_schema(self):
""" Get the schema for the registered models """
schema = []
for model in six.itervalues(self.models):
schema.append(model.meta_.ddb_tablename)
schema.append(model.meta_.ddb_tablename(self.namespace))
return schema
def __call__(self, model):
@@ -243,8 +249,9 @@ def get(self, model, pkeys=None, consistent=False, **kwargs):
else:
keys = [model.meta_.pk_dict(scope=kwargs)]
raw_items = self.dynamo.batch_get(model.meta_.ddb_tablename, keys,
consistent=consistent)
raw_items = self.dynamo.batch_get(
model.meta_.ddb_tablename(self.namespace), keys,
consistent=consistent)
items = [model.ddb_load_(self, raw_item) for raw_item in raw_items]
if pkeys is not None:
return items
@@ -289,7 +296,8 @@ def delete_key(self, model, pkeys=None, **kwargs):
keys = [kwargs]
count = 0
with self.dynamo.batch_write(model.meta_.ddb_tablename) as batch:
tablename = model.meta_.ddb_tablename(self.namespace)
with self.dynamo.batch_write(tablename) as batch:
for key in keys:
pkey = model.meta_.pk_dict(scope=key)
batch.delete(pkey)
@@ -331,7 +339,7 @@ def delete(self, items, raise_on_conflict=None):
return
tables = defaultdict(list)
for item in items:
tables[item.meta_.ddb_tablename].append(item)
tables[item.meta_.ddb_tablename(self.namespace)].append(item)
count = 0
for tablename, items in six.iteritems(tables):
@@ -386,7 +394,7 @@ def save(self, items, overwrite=None):
return
tables = defaultdict(list)
for item in items:
tables[item.meta_.ddb_tablename].append(item)
tables[item.meta_.ddb_tablename(self.namespace)].append(item)
for tablename, items in six.iteritems(tables):
if overwrite:
with self.dynamo.batch_write(tablename) as batch:
@@ -421,7 +429,7 @@ def refresh(self, items, consistent=False):
tables = defaultdict(list)
for item in items:
tables[item.meta_.ddb_tablename].append(item)
tables[item.meta_.ddb_tablename(self.namespace)].append(item)
for tablename, items in six.iteritems(tables):
keys = [item.pk_dict_ for item in items]
@@ -517,9 +525,9 @@ def sync(self, items, raise_on_conflict=None, consistent=False):
updates.append(update)
# Perform sync
ret = self.dynamo.update_item(item.meta_.ddb_tablename,
item.pk_dict_, updates,
returns=ALL_NEW)
ret = self.dynamo.update_item(
item.meta_.ddb_tablename(self.namespace), item.pk_dict_,
updates, returns=ALL_NEW)
# Load updated data back into object
with item.loading_(self):
@@ -117,8 +117,6 @@ class ModelMetadata(object):
__metadata__. Defaults to the name of the model class.
abstract : bool
If a model is abstract then it has no table in Dynamo
namespace : list
The namespace of this model. Set by the Engine.
global_indexes : list
List of global indexes (hash_key, [range_key]) pairs.
related_fields : dict
@@ -132,7 +130,6 @@ class ModelMetadata(object):
"""
__order_class__ = Ordering
namespace = []
def __init__(self, model):
self.model = model
@@ -317,12 +314,24 @@ def abstract(self):
""" Getter for abstract """
return self._abstract
@property
def ddb_tablename(self):
""" The name of the DynamoDB table """
def ddb_tablename(self, namespace=()):
"""
The name of the DynamoDB table
Parameters
----------
namespace : list or str, optional
String prefix or list of component parts of a prefix for the table
name. The prefix will be this string or strings (joined by '-').
"""
if isinstance(namespace, six.string_types):
namespace = (namespace,)
else:
namespace = tuple(namespace)
if self.abstract:
return None
return '-'.join(self.namespace + [self.name])
return '-'.join(namespace + (self.name,))
def validate_model(self):
""" Perform validation checks on the model declaration """
@@ -359,7 +368,7 @@ def validate_model(self):
"itself" % (name, field.name))
def create_dynamo_schema(self, connection, tablenames=None, test=False,
wait=False, throughput=None):
wait=False, throughput=None, namespace=()):
"""
Create all Dynamo tables for this model
@@ -378,6 +387,8 @@ def create_dynamo_schema(self, connection, tablenames=None, test=False,
and 'write'. To specify throughput for global indexes, add the name
of the index as a key and another 'read', 'write' dict as the
value.
namespace : tuple, optional
The namespace of the table
Returns
-------
@@ -389,10 +400,11 @@ def create_dynamo_schema(self, connection, tablenames=None, test=False,
return None
if tablenames is None:
tablenames = set(connection.list_tables())
if self.ddb_tablename in tablenames:
tablename = self.ddb_tablename(namespace)
if tablename in tablenames:
return None
elif test:
return self.ddb_tablename
return tablename
indexes = []
global_indexes = []
@@ -422,18 +434,18 @@ def create_dynamo_schema(self, connection, tablenames=None, test=False,
global_indexes.append(index)
if not test:
connection.create_table(self.ddb_tablename, hash_key, range_key,
connection.create_table(tablename, hash_key, range_key,
indexes, global_indexes, table_throughput)
if wait:
desc = connection.describe_table(self.ddb_tablename)
desc = connection.describe_table(tablename)
while desc.status != 'ACTIVE':
time.sleep(1)
desc = connection.describe_table(self.ddb_tablename)
desc = connection.describe_table(tablename)
return self.ddb_tablename
return tablename
def delete_dynamo_schema(self, connection, tablenames=None, test=False,
wait=False):
wait=False, namespace=()):
"""
Drop all Dynamo tables for this model
@@ -447,6 +459,8 @@ def delete_dynamo_schema(self, connection, tablenames=None, test=False,
If True, don't actually delete the table (default False)
wait : bool, optional
If True, block until table has been deleted (default False)
namespace : tuple, optional
The namespace of the table
Returns
-------
@@ -459,12 +473,13 @@ def delete_dynamo_schema(self, connection, tablenames=None, test=False,
if tablenames is None:
tablenames = set(connection.list_tables())
if self.ddb_tablename in tablenames:
tablename = self.ddb_tablename(namespace)
if tablename in tablenames:
if not test:
connection.delete_table(self.ddb_tablename)
connection.delete_table(tablename)
if wait:
desc = connection.describe_table(self.ddb_tablename)
desc = connection.describe_table(tablename)
while desc is not None:
desc = connection.describe_table(self.ddb_tablename)
return self.ddb_tablename
desc = connection.describe_table(tablename)
return tablename
return None
@@ -32,7 +32,7 @@ def dynamo(self):
@property
def tablename(self):
""" Shortcut to access dynamo table name """
return self.model.meta_.ddb_tablename
return self.model.meta_.ddb_tablename(self.engine.namespace)
def gen(self, desc=False, consistent=False, attributes=None):
"""
@@ -330,7 +330,8 @@ def test_save_defaults(self):
""" Default field values are saved to dynamo """
w = Widget(string2='abc')
self.engine.sync(w)
result = six.next(self.dynamo.scan(Widget.meta_.ddb_tablename))
tablename = Widget.meta_.ddb_tablename(self.engine.namespace)
result = six.next(self.dynamo.scan(tablename))
self.assertEquals(result, {
'string': w.string,
'string2': w.string2,
@@ -372,7 +373,8 @@ def test_store_extra_number(self):
w = Widget(string='a', foobar=5)
self.engine.sync(w)
result = six.next(self.dynamo.scan(Widget.meta_.ddb_tablename))
tablename = Widget.meta_.ddb_tablename(self.engine.namespace)
result = six.next(self.dynamo.scan(tablename))
self.assertEquals(result['foobar'], 5)
stored_widget = self.engine.scan(Widget).all()[0]
self.assertEquals(stored_widget.foobar, 5)
@@ -382,7 +384,8 @@ def test_store_extra_string(self):
w = Widget(string='a', foobar='hi')
self.engine.sync(w)
result = six.next(self.dynamo.scan(Widget.meta_.ddb_tablename))
tablename = Widget.meta_.ddb_tablename(self.engine.namespace)
result = six.next(self.dynamo.scan(tablename))
self.assertEquals(result['foobar'], json.dumps('hi'))
stored_widget = self.engine.scan(Widget).all()[0]
self.assertEquals(stored_widget.foobar, 'hi')
@@ -393,7 +396,8 @@ def test_store_extra_set(self):
w = Widget(string='a', foobar=foobar)
self.engine.sync(w)
result = six.next(self.dynamo.scan(Widget.meta_.ddb_tablename))
tablename = Widget.meta_.ddb_tablename(self.engine.namespace)
result = six.next(self.dynamo.scan(tablename))
self.assertEquals(result['foobar'], foobar)
stored_widget = self.engine.scan(Widget).all()[0]
self.assertEquals(stored_widget.foobar, foobar)
@@ -404,7 +408,8 @@ def test_store_extra_dict(self):
w = Widget(string='a', foobar=foobar)
self.engine.save(w)
result = six.next(self.dynamo.scan(Widget.meta_.ddb_tablename))
tablename = Widget.meta_.ddb_tablename(self.engine.namespace)
result = six.next(self.dynamo.scan(tablename))
self.assertEquals(result['foobar'], json.dumps(foobar))
stored_widget = self.engine.scan(Widget).all()[0]
self.assertEquals(stored_widget.foobar, foobar)
Oops, something went wrong.

0 comments on commit 3b4fad7

Please sign in to comment.