Permalink
Browse files

Adding support for BatchGetItem to Layer2. Closes #533.

  • Loading branch information...
1 parent 4f148ea commit 9d42dea0683117a537046cdae7b0fa5d7145a386 @garnaat garnaat committed Jan 30, 2012
@@ -0,0 +1,76 @@
+# Copyright (c) 2012 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+
+class Batch(object):
+ """
+ :ivar table: The Table object from which the item is retrieved.
+
+ :ivar keys: A list of scalar or tuple values. Each element in the
+ list represents one Item to retrieve. If the schema for the
+ table has both a HashKey and a RangeKey, each element in the
+ list should be a tuple consisting of (hash_key, range_key). If
+ the schema for the table contains only a HashKey, each element
+ in the list should be a scalar value of the appropriate type
+ for the table schema.
+
+ :ivar attributes_to_get: A list of attribute names.
+ If supplied, only the specified attribute names will
+ be returned. Otherwise, all attributes will be returned.
+ """
+
+ def __init__(self, table, keys, attributes_to_get=None):
+ self.table = table
+ self.keys = keys
+ self.attributes_to_get = attributes_to_get
+
+class BatchList(list):
+ """
+ A subclass of a list object that contains a collection of
+ :class:`boto.dynamodb.batch.Batch` objects.
+ """
+
+ def add_batch(self, table, keys, attributes_to_get=None):
+ """
+ Add a Batch to this BatchList.
+
+ :type table: :class:`boto.dynamodb.table.Table`
+ :param table: The Table object in which the items are contained.
+
+ :type keys: list
+ :param keys: A list of scalar or tuple values. Each element in the
+ list represents one Item to retrieve. If the schema for the
+ table has both a HashKey and a RangeKey, each element in the
+ list should be a tuple consisting of (hash_key, range_key). If
+ the schema for the table contains only a HashKey, each element
+ in the list should be a scalar value of the appropriate type
+ for the table schema.
+
+ :type attributes_to_get: list
+ :param attributes_to_get: A list of attribute names.
+ If supplied, only the specified attribute names will
+ be returned. Otherwise, all attributes will be returned.
+ """
+ self.append(Batch(table, keys, attributes_to_get))
+
+
+
@@ -273,7 +273,7 @@ def get_item(self, table_name, key, attributes_to_get=None,
)
return response
- def batch_get_item(self, request_items):
+ def batch_get_item(self, request_items, object_hook=None):
"""
Return a set of attributes for a multiple items in
multiple tables using their primary keys.
@@ -284,7 +284,8 @@ def batch_get_item(self, request_items):
"""
data = {'RequestItems' : request_items}
json_input = json.dumps(data)
- return self.make_request('BatchGetItem', json_input)
+ return self.make_request('BatchGetItem', json_input,
+ object_hook=object_hook)
def put_item(self, table_name, item,
expected=None, return_values=None):
@@ -459,7 +460,8 @@ def query(self, table_name, hash_key_value, range_key_conditions=None,
def scan(self, table_name, scan_filter=None,
attributes_to_get=None, limit=None,
- count=False, exclusive_start_key=None):
+ count=False, exclusive_start_key=None,
+ object_hook=None):
"""
Perform a scan of DynamoDB. This version is currently punting
and expecting you to provide a full and correct JSON body
@@ -502,6 +504,6 @@ def scan(self, table_name, scan_filter=None,
if exclusive_start_key:
data['ExclusiveStartKey'] = exclusive_start_key
json_input = json.dumps(data)
- return self.make_request('Scan', json_input)
+ return self.make_request('Scan', json_input, object_hook=object_hook)
@@ -25,6 +25,7 @@
from boto.dynamodb.table import Table
from boto.dynamodb.schema import Schema
from boto.dynamodb.item import Item
+from boto.dynamodb.batch import BatchList
"""
Some utility functions to deal with mapping Amazon DynamoDB types to
@@ -120,6 +121,32 @@ def dynamize_expected_value(self, expected_value):
d[attr_name] = attr_value
return d
+ def dynamize_request_items(self, batch_list):
+ """
+ Convert a request_items parameter into the data structure
+ required for Layer1.
+ """
+ d = None
+ if batch_list:
+ d = {}
+ for batch in batch_list:
+ batch_dict = {}
+ key_list = []
+ for key in batch.keys:
+ if isinstance(key, tuple):
+ hash_key, range_key = key
+ else:
+ hash_key = key
+ range_key = None
+ k = self.build_key_from_values(batch.table.schema,
+ hash_key, range_key)
+ key_list.append(k)
+ batch_dict['Keys'] = key_list
+ if batch.attributes_to_get:
+ batch_dict['AttributesToGet'] = batch.attributes_to_get
+ d[batch.table.name] = batch_dict
+ return d
+
def get_dynamodb_type(self, val):
"""
Take a scalar Python value and return a string representing
@@ -201,6 +228,9 @@ def build_key_from_values(self, schema, hash_key, range_key=None):
dynamodb_key['RangeKeyElement'] = dynamodb_value
return dynamodb_key
+ def new_batch_list(self):
+ return BatchList()
+
def list_tables(self, limit=None, start_table=None):
"""
Return a list of the names of all Tables associated with the
@@ -376,6 +406,22 @@ def get_item(self, table, hash_key, range_key=None,
item.consumed_units = response['ConsumedCapacityUnits']
return item
+ def batch_get_item(self, batch_list):
+ """
+ Return a set of attributes for a multiple items in
+ multiple tables using their primary keys.
+
+ :type batch_list: :class:`boto.dynamodb.batch.BatchList`
+ :param batch_list: A BatchList object which consists of a
+ list of :class:`boto.dynamoddb.batch.Batch` objects.
+ Each Batch object contains the information about one
+ batch of objects that you wish to retrieve in this
+ request.
+ """
+ request_items = self.dynamize_request_items(batch_list)
+ return self.layer1.batch_get_item(request_items,
+ object_hook=item_object_hook)
+
def put_item(self, item, expected_value=None, return_values=None):
"""
Store a new item or completely replace an existing item
@@ -551,8 +597,9 @@ def scan(self, table, scan_filter=None,
break
response = self.layer1.scan(table.name, scan_filter,
- attributes_to_get,limit,
- count, exclusive_start_key)
+ attributes_to_get,limit,
+ count, exclusive_start_key,
+ object_hook=item_object_hook)
if response:
for item in response['Items']:
yield item_class(table, attrs=item)
@@ -179,9 +179,10 @@ def get_item(self, hash_key, range_key=None,
:class:`boto.dynamodb.item.Item`
"""
return self.layer2.get_item(self, hash_key, range_key,
- attributes_to_get, consistent_read, item_class)
-
+ attributes_to_get, consistent_read,
+ item_class)
lookup = get_item
+
def has_item(self, hash_key, range_key=None, consistent_read=False):
"""
Checks the table to see if the Item with the specified ``hash_key``
@@ -288,9 +289,10 @@ def scan(self, scan_filter=None,
attributes_to_get=None, limit=None,
count=False, exclusive_start_key=None,
item_class=Item):
- """Scan through this table, this is a very long
+ """
+ Scan through this table, this is a very long
and expensive operation, and should be avoided if
- at all possible
+ at all possible.
:type scan_filter: dict
:param scan_filter: A Python version of the
@@ -46,4 +46,11 @@ boto.dynamodb.item
:members:
:undoc-members:
+boto.dynamodb.batch
+-------------------
+
+.. automodule:: boto.dynamodb.batch
+ :members:
+ :undoc-members:
+
@@ -248,6 +248,13 @@ def test_layer2_basic(self):
assert i in mixed_set
for i in item4['StrSetAttr']:
assert i in str_set
+
+ # Try a batch get
+ batch_list = c.new_batch_list()
+ batch_list.add_batch(table, [(item2_key, item2_range),
+ (item3_key, item3_range)])
+ response = c.batch_get_item(batch_list)
+ assert len(response['Responses'][table.name]['Items']) == 2
# Try to delete the item with the right Expected value
expected = {'Views': 0}

0 comments on commit 9d42dea

Please sign in to comment.