Skip to content
This repository has been archived by the owner on May 10, 2024. It is now read-only.

Commit

Permalink
Merge pull request #2925 from pauloubuntu/develop
Browse files Browse the repository at this point in the history 
Add Amazon DynamoDB online indexing support on High level API
  • Loading branch information
@kyleknap
kyleknap committed Feb 6, 2015
2 parents 34a0f63 + 57c4e96 commit 0621c53
Show file tree
Hide file tree
Showing 3 changed files with 430 additions and 26 deletions.
214 changes: 189 additions & 25 deletions boto/dynamodb2/table.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,9 +137,9 @@ def create(cls, table_name, schema, throughput=None, indexes=None,
to define the key structure of the table.
**IMPORTANT** - You should consider the usage pattern of your table
up-front, as the schema & indexes can **NOT** be modified once the
table is created, requiring the creation of a new table & migrating
the data should you wish to revise it.
up-front, as the schema can **NOT** be modified once the table is
created, requiring the creation of a new table & migrating the data
should you wish to revise it.
**IMPORTANT** - If the table already exists in DynamoDB, additional
calls to this method will result in an error. If you just need
Expand Down Expand Up @@ -371,25 +371,27 @@ def describe(self):
raw_indexes = result['Table'].get('LocalSecondaryIndexes', [])
self.indexes = self._introspect_indexes(raw_indexes)

if not self.global_indexes:
# Build the global index information as well.
raw_global_indexes = result['Table'].get('GlobalSecondaryIndexes', [])
self.global_indexes = self._introspect_global_indexes(raw_global_indexes)
# Build the global index information as well.
raw_global_indexes = result['Table'].get('GlobalSecondaryIndexes', [])
self.global_indexes = self._introspect_global_indexes(raw_global_indexes)

# This is leaky.
return result

def update(self, throughput, global_indexes=None):
def update(self, throughput=None, global_indexes=None):
"""
Updates table attributes in DynamoDB.
Updates table attributes and global indexes in DynamoDB.
Currently, the only thing you can modify about a table after it has
been created is the throughput.
Requires a ``throughput`` parameter, which should be a
Optionally accepts a ``throughput`` parameter, which should be a
dictionary. If provided, it should specify a ``read`` & ``write`` key,
both of which should have an integer value associated with them.
Optionally accepts a ``global_indexes`` parameter, which should be a
dictionary. If provided, it should specify the index name, which is also
a dict containing a ``read`` & ``write`` key, both of which
should have an integer value associated with them. If you are writing
new code, please use ``Table.update_global_secondary_index``.
Returns ``True`` on success.
Example::
Expand All @@ -413,13 +415,17 @@ def update(self, throughput, global_indexes=None):
... }
... })
True
"""
self.throughput = throughput
data = {
'ReadCapacityUnits': int(self.throughput['read']),
'WriteCapacityUnits': int(self.throughput['write']),
}

data = None

if throughput:
self.throughput = throughput
data = {
'ReadCapacityUnits': int(self.throughput['read']),
'WriteCapacityUnits': int(self.throughput['write']),
}

gsi_data = None

if global_indexes:
Expand All @@ -436,12 +442,170 @@ def update(self, throughput, global_indexes=None):
},
})

self.connection.update_table(
self.table_name,
provisioned_throughput=data,
global_secondary_index_updates=gsi_data
)
return True
if throughput or global_indexes:
self.connection.update_table(
self.table_name,
provisioned_throughput=data,
global_secondary_index_updates=gsi_data,
)

return True
else:
msg = 'You need to provide either the throughput or the ' \
'global_indexes to update method'
boto.log.error(msg)

return False

def create_global_secondary_index(self, global_index):
"""
Creates a global index in DynamoDB after the table has been created.
Requires a ``global_indexes`` parameter, which should be a
``GlobalBaseIndexField`` subclass representing the desired index.
To update ``global_indexes`` information on the ``Table``, you'll need
to call ``Table.describe``.
Returns ``True`` on success.
Example::
# To create a global index
>>> users.create_global_secondary_index(
... global_index=GlobalAllIndex(
... 'TheIndexNameHere', parts=[
... HashKey('requiredHashkey', data_type=STRING),
... RangeKey('optionalRangeKey', data_type=STRING)
... ],
... throughput={
... 'read': 2,
... 'write': 1,
... })
... )
True
"""

if global_index:
gsi_data = []
gsi_data_attr_def = []

gsi_data.append({
"Create": global_index.schema()
})

for attr_def in global_index.parts:
gsi_data_attr_def.append(attr_def.definition())

self.connection.update_table(
self.table_name,
global_secondary_index_updates=gsi_data,
attribute_definitions=gsi_data_attr_def
)

return True
else:
msg = 'You need to provide the global_index to ' \
'create_global_secondary_index method'
boto.log.error(msg)

return False

def delete_global_secondary_index(self, global_index_name):
"""
Deletes a global index in DynamoDB after the table has been created.
Requires a ``global_index_name`` parameter, which should be a simple
string of the name of the global secondary index.
To update ``global_indexes`` information on the ``Table``, you'll need
to call ``Table.describe``.
Returns ``True`` on success.
Example::
# To delete a global index
>>> users.delete_global_secondary_index('TheIndexNameHere')
True
"""

if global_index_name:
gsi_data = [
{
"Delete": {
"IndexName": global_index_name
}
}
]

self.connection.update_table(
self.table_name,
global_secondary_index_updates=gsi_data,
)

return True
else:
msg = 'You need to provide the global index name to ' \
'delete_global_secondary_index method'
boto.log.error(msg)

return False

def update_global_secondary_index(self, global_indexes):
"""
Updates a global index(es) in DynamoDB after the table has been created.
Requires a ``global_indexes`` parameter, which should be a
dictionary. If provided, it should specify the index name, which is also
a dict containing a ``read`` & ``write`` key, both of which
should have an integer value associated with them.
To update ``global_indexes`` information on the ``Table``, you'll need
to call ``Table.describe``.
Returns ``True`` on success.
Example::
# To update a global index
>>> users.update_global_secondary_index(global_indexes={
... 'TheIndexNameHere': {
... 'read': 15,
... 'write': 5,
... }
... })
True
"""

if global_indexes:
gsi_data = []

for gsi_name, gsi_throughput in global_indexes.items():
gsi_data.append({
"Update": {
"IndexName": gsi_name,
"ProvisionedThroughput": {
"ReadCapacityUnits": int(gsi_throughput['read']),
"WriteCapacityUnits": int(gsi_throughput['write']),
},
},
})

self.connection.update_table(
self.table_name,
global_secondary_index_updates=gsi_data,
)
return True
else:
msg = 'You need to provide the global indexes to ' \
'update_global_secondary_index method'
boto.log.error(msg)

return False

def delete(self):
"""
Expand Down
105 changes: 104 additions & 1 deletion tests/integration/dynamodb2/test_highlevel.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
GlobalAllIndex)
from boto.dynamodb2.items import Item
from boto.dynamodb2.table import Table
from boto.dynamodb2.types import NUMBER
from boto.dynamodb2.types import NUMBER, STRING

try:
import json
Expand Down Expand Up @@ -716,3 +716,106 @@ def test_query_after_describe_with_gsi(self):

for rs_item in rs:
self.assertEqual(rs_item['username'], ['johndoe'])

def test_update_table_online_indexing_support(self):
# Create a table using gsi to test the DynamoDB online indexing support
# https://github.com/boto/boto/pull/2925
users = Table.create('online_indexing_support_users', schema=[
HashKey('user_id')
], throughput={
'read': 5,
'write': 5
}, global_indexes=[
GlobalAllIndex('EmailGSIIndex', parts=[
HashKey('email')
], throughput={
'read': 2,
'write': 2
})
])

# Add this function to be called after tearDown()
self.addCleanup(users.delete)

# Wait for it.
time.sleep(60)

# Fetch fresh table desc from DynamoDB
users.describe()

# Assert if everything is fine so far
self.assertEqual(len(users.global_indexes), 1)
self.assertEqual(users.global_indexes[0].throughput['read'], 2)
self.assertEqual(users.global_indexes[0].throughput['write'], 2)

# Update a GSI throughput. it should work.
users.update_global_secondary_index(global_indexes={
'EmailGSIIndex': {
'read': 2,
'write': 1,
}
})

# Wait for it.
time.sleep(60)

# Fetch fresh table desc from DynamoDB
users.describe()

# Assert if everything is fine so far
self.assertEqual(len(users.global_indexes), 1)
self.assertEqual(users.global_indexes[0].throughput['read'], 2)
self.assertEqual(users.global_indexes[0].throughput['write'], 1)

# Update a GSI throughput using the old fashion way for compatibility
# purposes. it should work.
users.update(global_indexes={
'EmailGSIIndex': {
'read': 3,
'write': 2,
}
})

# Wait for it.
time.sleep(60)

# Fetch fresh table desc from DynamoDB
users.describe()

# Assert if everything is fine so far
self.assertEqual(len(users.global_indexes), 1)
self.assertEqual(users.global_indexes[0].throughput['read'], 3)
self.assertEqual(users.global_indexes[0].throughput['write'], 2)

# Delete a GSI. it should work.
users.delete_global_secondary_index('EmailGSIIndex')

# Wait for it.
time.sleep(60)

# Fetch fresh table desc from DynamoDB
users.describe()

# Assert if everything is fine so far
self.assertEqual(len(users.global_indexes), 0)

# Create a GSI. it should work.
users.create_global_secondary_index(
global_index=GlobalAllIndex(
'AddressGSIIndex', parts=[
HashKey('address', data_type=STRING)
], throughput={
'read': 1,
'write': 1,
})
)
# Wait for it. This operation usually takes much longer than the others
time.sleep(60*10)

# Fetch fresh table desc from DynamoDB
users.describe()

# Assert if everything is fine so far
self.assertEqual(len(users.global_indexes), 1)
self.assertEqual(users.global_indexes[0].throughput['read'], 1)
self.assertEqual(users.global_indexes[0].throughput['write'], 1)
Loading

0 comments on commit 0621c53

Please sign in to comment.