From a1b58e4011c24f1593ae89227e80baa231177e23 Mon Sep 17 00:00:00 2001 From: Vincent Jacques Date: Sun, 26 Apr 2015 23:17:59 +0200 Subject: [PATCH] Make all compounds functions This changes the public interface but this is good: This solves a naming issue (BatchPutItem was a function named like a class to be consistent with BatchGetItemIterator. BatchGetItemIterator was a class only because of an implementation detail) Conceptually, it's always been clear that compounds receive the connection as an argument and therefore were already *used as* functions. Now they *are* functions. --- LowVoltage/actions/batch_get_item.py | 8 +- LowVoltage/actions/batch_write_item.py | 4 +- LowVoltage/actions/create_table.py | 18 +-- LowVoltage/actions/delete_table.py | 6 +- LowVoltage/actions/list_tables.py | 6 +- LowVoltage/actions/query.py | 6 +- LowVoltage/actions/scan.py | 8 +- LowVoltage/actions/update_table.py | 28 ++--- LowVoltage/compounds/__init__.py | 16 +-- LowVoltage/compounds/batch_delete_item.py | 22 ++-- LowVoltage/compounds/batch_put_item.py | 18 +-- ..._iterator.py => iterate_batch_get_item.py} | 78 ++++++------- ...les_iterator.py => iterate_list_tables.py} | 43 ++++---- .../{query_iterator.py => iterate_query.py} | 29 ++--- LowVoltage/compounds/iterate_scan.py | 95 ++++++++++++++++ LowVoltage/compounds/iterator.py | 31 ------ LowVoltage/compounds/scan_iterator.py | 103 ------------------ LowVoltage/compounds/tests/integ/local.py | 8 +- LowVoltage/compounds/tests/unit.py | 4 +- .../compounds/wait_for_table_activation.py | 10 +- .../compounds/wait_for_table_deletion.py | 12 +- LowVoltage/testing/connected_integ_tests.py | 4 +- LowVoltage/testing/doc_tests.py | 12 +- doc/changelog.rst | 7 +- doc/reference.rst | 8 +- doc/reference/compounds/batch_delete_item.rst | 4 +- .../compounds/batch_get_item_iterator.rst | 4 - doc/reference/compounds/batch_put_item.rst | 4 +- .../compounds/iterate_batch_get_item.rst | 4 + .../compounds/iterate_list_tables.rst | 4 + doc/reference/compounds/iterate_query.rst | 4 + doc/reference/compounds/iterate_scan.rst | 4 + .../compounds/list_tables_iterator.rst | 4 - doc/reference/compounds/query_iterator.rst | 4 - doc/reference/compounds/scan_iterator.rst | 4 - .../compounds/wait_for_table_activation.rst | 4 +- .../compounds/wait_for_table_deletion.rst | 4 +- doc/user_guide.rst | 10 +- setup.py | 2 +- 39 files changed, 294 insertions(+), 350 deletions(-) rename LowVoltage/compounds/{batch_get_item_iterator.py => iterate_batch_get_item.py} (66%) rename LowVoltage/compounds/{list_tables_iterator.py => iterate_list_tables.py} (68%) rename LowVoltage/compounds/{query_iterator.py => iterate_query.py} (60%) create mode 100644 LowVoltage/compounds/iterate_scan.py delete mode 100644 LowVoltage/compounds/iterator.py delete mode 100644 LowVoltage/compounds/scan_iterator.py delete mode 100644 doc/reference/compounds/batch_get_item_iterator.rst create mode 100644 doc/reference/compounds/iterate_batch_get_item.rst create mode 100644 doc/reference/compounds/iterate_list_tables.rst create mode 100644 doc/reference/compounds/iterate_query.rst create mode 100644 doc/reference/compounds/iterate_scan.rst delete mode 100644 doc/reference/compounds/list_tables_iterator.rst delete mode 100644 doc/reference/compounds/query_iterator.rst delete mode 100644 doc/reference/compounds/scan_iterator.rst diff --git a/LowVoltage/actions/batch_get_item.py b/LowVoltage/actions/batch_get_item.py index 05d91b1..e2b871f 100644 --- a/LowVoltage/actions/batch_get_item.py +++ b/LowVoltage/actions/batch_get_item.py @@ -17,7 +17,7 @@ Note that responses are in an undefined order. -See also the :class:`.BatchGetItemIterator` compound. And :ref:`actions-vs-compounds` in the user guide. +See also the :func:`.iterate_batch_get_item` compound. And :ref:`actions-vs-compounds` in the user guide. """ import LowVoltage as _lv @@ -75,7 +75,7 @@ def unprocessed_keys(self): """ Keys that were not processed during this request. If not None, you should give this back to the constructor of a subsequent :class:`BatchGetItem`. - The :class:`.BatchGetItemIterator` compound processes those for you. + The :func:`.iterate_batch_get_item` compound processes those for you. :type: ``None`` or exactly as returned by DynamoDB """ @@ -418,7 +418,7 @@ def test_get_unexisting_keys(self): self.assertEqual(r.unprocessed_keys, {}) def test_get_without_unprocessed_keys(self): - _lv.BatchPutItem(self.connection, "Aaa", [{"h": unicode(i)} for i in range(100)]) + _lv.batch_put_item(self.connection, "Aaa", [{"h": unicode(i)} for i in range(100)]) r = self.connection(_lv.BatchGetItem().table("Aaa").keys({"h": unicode(i)} for i in range(100))) @@ -426,7 +426,7 @@ def test_get_without_unprocessed_keys(self): self.assertEqual(len(r.responses["Aaa"]), 100) def test_get_with_unprocessed_keys(self): - _lv.BatchPutItem(self.connection, "Aaa", [{"h": unicode(i), "xs": "x" * 300000} for i in range(100)]) # 300kB items ensure a single BatchGetItem will return at most 55 items + _lv.batch_put_item(self.connection, "Aaa", [{"h": unicode(i), "xs": "x" * 300000} for i in range(100)]) # 300kB items ensure a single BatchGetItem will return at most 55 items r1 = self.connection(_lv.BatchGetItem().table("Aaa").keys({"h": unicode(i)} for i in range(100))) diff --git a/LowVoltage/actions/batch_write_item.py b/LowVoltage/actions/batch_write_item.py index d3c160c..f8e1acc 100644 --- a/LowVoltage/actions/batch_write_item.py +++ b/LowVoltage/actions/batch_write_item.py @@ -10,7 +10,7 @@ ... ) -See also the :func:`.BatchPutItem` and :func:`.BatchDeleteItem` compounds. And :ref:`actions-vs-compounds` in the user guide. +See also the :func:`.batch_put_item` and :func:`.batch_delete_item` compounds. And :ref:`actions-vs-compounds` in the user guide. """ import LowVoltage as _lv @@ -65,7 +65,7 @@ def unprocessed_items(self): """ Items that were not processed during this request. If not None, you should give this back to the constructor of a subsequent :class:`BatchWriteItem`. - The :func:`.BatchPutItem` and :func:`.BatchDeleteItem` compounds process those for you. + The :func:`.batch_put_item` and :func:`.batch_delete_item` compounds process those for you. :type: ``None`` or exactly as returned by DynamoDB """ diff --git a/LowVoltage/actions/create_table.py b/LowVoltage/actions/create_table.py index 3dff376..fb685d2 100644 --- a/LowVoltage/actions/create_table.py +++ b/LowVoltage/actions/create_table.py @@ -22,22 +22,22 @@ >>> r.table_description.table_status u'CREATING' -Note that you can use the :func:`.WaitForTableActivation` compound to poll the table status until it's usable. See :ref:`actions-vs-compounds` in the user guide. +Note that you can use the :func:`.wait_for_table_activation` compound to poll the table status until it's usable. See :ref:`actions-vs-compounds` in the user guide. .. testcleanup:: - WaitForTableActivation(connection, table) - WaitForTableActivation(connection, table2) - WaitForTableActivation(connection, table3) - WaitForTableActivation(connection, table4) + wait_for_table_activation(connection, table) + wait_for_table_activation(connection, table2) + wait_for_table_activation(connection, table3) + wait_for_table_activation(connection, table4) connection(DeleteTable(table)) connection(DeleteTable(table2)) connection(DeleteTable(table3)) connection(DeleteTable(table4)) - WaitForTableDeletion(connection, table) - WaitForTableDeletion(connection, table2) - WaitForTableDeletion(connection, table3) - WaitForTableDeletion(connection, table4) + wait_for_table_deletion(connection, table) + wait_for_table_deletion(connection, table2) + wait_for_table_deletion(connection, table3) + wait_for_table_deletion(connection, table4) """ import datetime diff --git a/LowVoltage/actions/delete_table.py b/LowVoltage/actions/delete_table.py index 791331e..67a9dde 100644 --- a/LowVoltage/actions/delete_table.py +++ b/LowVoltage/actions/delete_table.py @@ -9,7 +9,7 @@ table = "LowVoltage.Tests.Doc.DeleteTable.1" connection(CreateTable(table).hash_key("h", STRING).provisioned_throughput(1, 1)) - WaitForTableActivation(connection, table) + wait_for_table_activation(connection, table) >>> r = connection(DeleteTable(table)) >>> r @@ -17,11 +17,11 @@ >>> r.table_description.table_status u'DELETING' -Note that you can use the :func:`.WaitForTableDeletion` compound to poll the table status until it's deleted. See :ref:`actions-vs-compounds` in the user guide. +Note that you can use the :func:`.wait_for_table_deletion` compound to poll the table status until it's deleted. See :ref:`actions-vs-compounds` in the user guide. .. testcleanup:: - WaitForTableDeletion(connection, table) + wait_for_table_deletion(connection, table) """ import datetime diff --git a/LowVoltage/actions/list_tables.py b/LowVoltage/actions/list_tables.py index dded880..f0bace3 100644 --- a/LowVoltage/actions/list_tables.py +++ b/LowVoltage/actions/list_tables.py @@ -11,7 +11,7 @@ >>> r.table_names [u'LowVoltage.Tests.Doc.1', u'LowVoltage.Tests.Doc.2'] -See also the :class:`.ListTablesIterator` compound. And :ref:`actions-vs-compounds` in the user guide. +See also the :func:`.iterate_list_tables` compound. And :ref:`actions-vs-compounds` in the user guide. """ import LowVoltage as _lv @@ -40,7 +40,7 @@ def last_evaluated_table_name(self): The name of the last table that was considered during the request. If not None, you should give it to :meth:`~ListTables.exclusive_start_table_name` in a subsequent :class:`ListTables`. - The :class:`.ListTablesIterator` compound does that for you. + The :func:`.iterate_list_tables` compound does that for you. :type: ``None`` or string """ @@ -95,7 +95,7 @@ def exclusive_start_table_name(self, table_name): Set ExclusiveStartTableName. The response will contains tables that are after this one. Typically the :attr:`~ListTablesResponse.last_evaluated_table_name` of a previous response. - The :class:`.ListTablesIterator` compound does that for you. + The :func:`.iterate_list_tables` compound does that for you. >>> connection( ... ListTables() diff --git a/LowVoltage/actions/query.py b/LowVoltage/actions/query.py index e4e4819..63afa21 100644 --- a/LowVoltage/actions/query.py +++ b/LowVoltage/actions/query.py @@ -13,7 +13,7 @@ >>> connection(Query(table2).key_eq("h", 42)).items [{u'h': 42, u'r1': 0, u'r2': 10}, {u'h': 42, u'r1': 1, u'r2': 9}, {u'h': 42, u'r1': 2, u'r2': 8}, {u'h': 42, u'r1': 3, u'r2': 7}, {u'h': 42, u'r1': 4, u'r2': 6}, {u'h': 42, u'r1': 5, u'r2': 5}, {u'h': 42, u'r1': 6}, {u'h': 42, u'r1': 7}, {u'h': 42, u'r1': 8}, {u'h': 42, u'r1': 9}] -See also the :class:`.QueryIterator` compound. And :ref:`actions-vs-compounds` in the user guide. +See also the :func:`.iterate_query` compound. And :ref:`actions-vs-compounds` in the user guide. """ import LowVoltage as _lv @@ -91,7 +91,7 @@ def last_evaluated_key(self): """ The key of the last item evaluated by the query. If not None, it should be given to :meth:`~Query.exclusive_start_key` is a subsequent :class:`Query`. - The :class:`.QueryIterator` compound does that for you. + The :func:`.iterate_query` compound does that for you. :type: ``None`` or dict """ @@ -245,7 +245,7 @@ def key_between(self, name, lo, hi): @proxy("Query") def exclusive_start_key(self, key): """ - The :class:`.QueryIterator` compound does that for you. + The :func:`.iterate_query` compound does that for you. >>> r = connection( ... Query(table2) diff --git a/LowVoltage/actions/scan.py b/LowVoltage/actions/scan.py index df249a7..adc2346 100644 --- a/LowVoltage/actions/scan.py +++ b/LowVoltage/actions/scan.py @@ -15,7 +15,7 @@ Note that items are returned in an undefined order. -See also the :class:`.ScanIterator` compound. And :ref:`actions-vs-compounds` in the user guide. +See also the :func:`.iterate_scan` compound. And :ref:`actions-vs-compounds` in the user guide. """ import LowVoltage as _lv @@ -92,7 +92,7 @@ def last_evaluated_key(self): """ The key of the last item evaluated by the scan. If not None, it should be given to :meth:`~Scan.exclusive_start_key` is a subsequent :class:`Scan`. - The :class:`.ScanIterator` compound does that for you. + The :func:`.iterate_scan` compound does that for you. :type: ``None`` or dict """ @@ -151,7 +151,7 @@ def segment(self, segment, total_segments): Items will be partitioned in ``total_segments`` segments of approximately the same size, ans only the items of the ``segment``-th segment will be returned in this request. - :meth:`.ScanIterator.parallelize` does that for you. + :func:`.parallelize_scan` does that for you. >>> connection( ... Scan(table) @@ -172,7 +172,7 @@ def segment(self, segment, total_segments): @proxy("Scan") def exclusive_start_key(self, key): """ - The :class:`.ScanIterator` compound does that for you. + The :func:`.iterate_scan` compound does that for you. >>> r = connection( ... Scan(table) diff --git a/LowVoltage/actions/update_table.py b/LowVoltage/actions/update_table.py index 8f06229..c4d36f4 100644 --- a/LowVoltage/actions/update_table.py +++ b/LowVoltage/actions/update_table.py @@ -21,10 +21,10 @@ .global_secondary_index("gsi").hash_key("hh", STRING).range_key("rr", NUMBER).provisioned_throughput(1, 1).project_all() ) connection(CreateTable(table4).hash_key("h", STRING).provisioned_throughput(1, 1)) - WaitForTableActivation(connection, table) - WaitForTableActivation(connection, table2) - WaitForTableActivation(connection, table3) - WaitForTableActivation(connection, table4) + wait_for_table_activation(connection, table) + wait_for_table_activation(connection, table2) + wait_for_table_activation(connection, table3) + wait_for_table_activation(connection, table4) >>> r = connection( ... UpdateTable(table) @@ -35,22 +35,22 @@ >>> r.table_description.table_status u'UPDATING' -Note that you can use the :func:`.WaitForTableActivation` compound to poll the table status until it's updated. See :ref:`actions-vs-compounds` in the user guide. +Note that you can use the :func:`.wait_for_table_activation` compound to poll the table status until it's updated. See :ref:`actions-vs-compounds` in the user guide. .. testcleanup:: - WaitForTableActivation(connection, table) - WaitForTableActivation(connection, table2) - WaitForTableActivation(connection, table3) - WaitForTableActivation(connection, table4) + wait_for_table_activation(connection, table) + wait_for_table_activation(connection, table2) + wait_for_table_activation(connection, table3) + wait_for_table_activation(connection, table4) connection(DeleteTable(table)) connection(DeleteTable(table2)) connection(DeleteTable(table3)) connection(DeleteTable(table4)) - WaitForTableDeletion(connection, table) - WaitForTableDeletion(connection, table2) - WaitForTableDeletion(connection, table3) - WaitForTableDeletion(connection, table4) + wait_for_table_deletion(connection, table) + wait_for_table_deletion(connection, table2) + wait_for_table_deletion(connection, table3) + wait_for_table_deletion(connection, table4) """ import datetime @@ -582,7 +582,7 @@ def test_delete_and_create_gsi(self): r = self.connection(_lv.UpdateTable("Aaa").delete_global_secondary_index("the_gsi")) self.assertEqual(r.table_description.global_secondary_indexes[0].index_status, "DELETING") - _lv.WaitForTableActivation(self.connection, "Aaa") + _lv.wait_for_table_activation(self.connection, "Aaa") r = self.connection(_lv.DescribeTable("Aaa")) self.assertEqual(r.table.global_secondary_indexes, None) diff --git a/LowVoltage/compounds/__init__.py b/LowVoltage/compounds/__init__.py index 157c6ea..c5954f6 100644 --- a/LowVoltage/compounds/__init__.py +++ b/LowVoltage/compounds/__init__.py @@ -2,11 +2,11 @@ # Copyright 2014-2015 Vincent Jacques -from .batch_delete_item import BatchDeleteItem -from .batch_get_item_iterator import BatchGetItemIterator -from .batch_put_item import BatchPutItem -from .list_tables_iterator import ListTablesIterator -from .query_iterator import QueryIterator -from .scan_iterator import ScanIterator -from .wait_for_table_activation import WaitForTableActivation -from .wait_for_table_deletion import WaitForTableDeletion +from .batch_delete_item import batch_delete_item +from .iterate_batch_get_item import iterate_batch_get_item +from .batch_put_item import batch_put_item +from .iterate_list_tables import iterate_list_tables +from .iterate_query import iterate_query +from .iterate_scan import iterate_scan, parallelize_scan +from .wait_for_table_activation import wait_for_table_activation +from .wait_for_table_deletion import wait_for_table_deletion diff --git a/LowVoltage/compounds/batch_delete_item.py b/LowVoltage/compounds/batch_delete_item.py index 212736e..7a6b08a 100644 --- a/LowVoltage/compounds/batch_delete_item.py +++ b/LowVoltage/compounds/batch_delete_item.py @@ -8,12 +8,12 @@ @variadic(dict) -def BatchDeleteItem(connection, table, keys): +def batch_delete_item(connection, table, keys): """ Make as many :class:`.BatchWriteItem` actions as needed to delete all specified keys. Including processing :attr:`.BatchWriteItemResponse.unprocessed_items`. - >>> BatchDeleteItem( + >>> batch_delete_item( ... connection, ... table, ... {"h": 0}, @@ -30,7 +30,7 @@ def BatchDeleteItem(connection, table, keys): if isinstance(r.unprocessed_items, dict) and table in r.unprocessed_items: unprocessed_items += r.unprocessed_items[table] - # @todo Maybe wait a bit before retrying unprocessed items? Same in BatchPutItem and BatchGetItemIterator. + # @todo Maybe wait a bit before retrying unprocessed items? Same in batch_put_item and iterate_batch_get_item. # @todo In the first loop, maybe wait a bit before next request if we get unprocessed items? Might not be a good idea. while len(unprocessed_items) != 0: @@ -46,7 +46,7 @@ def setUp(self): self.connection = self.mocks.create("connection") def test_no_keys(self): - BatchDeleteItem(self.connection.object, "Aaa", []) + batch_delete_item(self.connection.object, "Aaa", []) def test_one_page(self): self.connection.expect._call_.withArguments( @@ -55,7 +55,7 @@ def test_one_page(self): _lv.BatchWriteItemResponse() ) - BatchDeleteItem(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"}) + batch_delete_item(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"}) def test_several_pages(self): self.connection.expect._call_.withArguments( @@ -74,7 +74,7 @@ def test_several_pages(self): _lv.BatchWriteItemResponse() ) - BatchDeleteItem(self.connection.object, "Aaa", ({"h": i} for i in range(60))) + batch_delete_item(self.connection.object, "Aaa", ({"h": i} for i in range(60))) def test_one_unprocessed_item(self): self.connection.expect._call_.withArguments( @@ -88,7 +88,7 @@ def test_one_unprocessed_item(self): _lv.BatchWriteItemResponse() ) - BatchDeleteItem(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"}) + batch_delete_item(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"}) def test_several_pages_of_unprocessed_item(self): self.connection.expect._call_.withArguments( @@ -112,7 +112,7 @@ def test_several_pages_of_unprocessed_item(self): _lv.BatchWriteItemResponse() ) - BatchDeleteItem(self.connection.object, "Aaa", [{"h": i} for i in range(35)]) + batch_delete_item(self.connection.object, "Aaa", [{"h": i} for i in range(35)]) class BatchDeleteItemLocalIntegTests(_tst.LocalIntegTestsWithTableH): @@ -121,8 +121,8 @@ def key(self, i): def setUp(self): super(BatchDeleteItemLocalIntegTests, self).setUp() - _lv.BatchPutItem(self.connection, "Aaa", [{"h": self.key(i)} for i in range(100)]) + _lv.batch_put_item(self.connection, "Aaa", [{"h": self.key(i)} for i in range(100)]) def test(self): - _lv.BatchDeleteItem(self.connection, "Aaa", [{"h": self.key(i)} for i in range(100)]) - self.assertEqual([], list(_lv.ScanIterator(self.connection, _lv.Scan("Aaa")))) + _lv.batch_delete_item(self.connection, "Aaa", [{"h": self.key(i)} for i in range(100)]) + self.assertEqual([], list(_lv.iterate_scan(self.connection, _lv.Scan("Aaa")))) diff --git a/LowVoltage/compounds/batch_put_item.py b/LowVoltage/compounds/batch_put_item.py index b29c0f5..e11b235 100644 --- a/LowVoltage/compounds/batch_put_item.py +++ b/LowVoltage/compounds/batch_put_item.py @@ -8,12 +8,12 @@ @variadic(dict) -def BatchPutItem(connection, table, items): +def batch_put_item(connection, table, items): """ Make as many :class:`.BatchWriteItem` actions as needed to put all specified items. Including processing :attr:`.BatchWriteItemResponse.unprocessed_items`. - >>> BatchPutItem( + >>> batch_put_item( ... connection, ... table, ... {"h": 0, "a": 42}, @@ -43,7 +43,7 @@ def setUp(self): self.connection = self.mocks.create("connection") def test_no_keys(self): - BatchPutItem(self.connection.object, "Aaa", []) + batch_put_item(self.connection.object, "Aaa", []) def test_one_page(self): self.connection.expect._call_.withArguments( @@ -52,7 +52,7 @@ def test_one_page(self): _lv.BatchWriteItemResponse() ) - BatchPutItem(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"}) + batch_put_item(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"}) def test_several_pages(self): self.connection.expect._call_.withArguments( @@ -71,7 +71,7 @@ def test_several_pages(self): _lv.BatchWriteItemResponse() ) - BatchPutItem(self.connection.object, "Aaa", ({"h": i} for i in range(60))) + batch_put_item(self.connection.object, "Aaa", ({"h": i} for i in range(60))) def test_one_unprocessed_item(self): self.connection.expect._call_.withArguments( @@ -85,7 +85,7 @@ def test_one_unprocessed_item(self): _lv.BatchWriteItemResponse() ) - BatchPutItem(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"}) + batch_put_item(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"}) def test_several_pages_of_unprocessed_item(self): self.connection.expect._call_.withArguments( @@ -109,7 +109,7 @@ def test_several_pages_of_unprocessed_item(self): _lv.BatchWriteItemResponse() ) - BatchPutItem(self.connection.object, "Aaa", [{"h": i} for i in range(35)]) + batch_put_item(self.connection.object, "Aaa", [{"h": i} for i in range(35)]) class BatchPutItemLocalIntegTests(_tst.LocalIntegTestsWithTableH): @@ -117,5 +117,5 @@ def key(self, i): return u"{:03}".format(i) def test(self): - _lv.BatchPutItem(self.connection, "Aaa", [{"h": self.key(i)} for i in range(100)]) - self.assertEqual(len(list(_lv.ScanIterator(self.connection, _lv.Scan("Aaa")))), 100) + _lv.batch_put_item(self.connection, "Aaa", [{"h": self.key(i)} for i in range(100)]) + self.assertEqual(len(list(_lv.iterate_scan(self.connection, _lv.Scan("Aaa")))), 100) diff --git a/LowVoltage/compounds/batch_get_item_iterator.py b/LowVoltage/compounds/iterate_batch_get_item.py similarity index 66% rename from LowVoltage/compounds/batch_get_item_iterator.py rename to LowVoltage/compounds/iterate_batch_get_item.py index de0ca86..d3252ef 100644 --- a/LowVoltage/compounds/batch_get_item_iterator.py +++ b/LowVoltage/compounds/iterate_batch_get_item.py @@ -5,13 +5,11 @@ import LowVoltage as _lv import LowVoltage.testing as _tst from LowVoltage.actions.next_gen_mixins import variadic -from .iterator import Iterator -class BatchGetItemIterator(Iterator): +@variadic(dict) +def iterate_batch_get_item(connection, table, keys): """ - Note that this function is variadic. See :ref:`variadic-functions`. - Make as many :class:`.BatchGetItem` actions as needed to iterate over all specified items. Including processing :attr:`.BatchGetItemResponse.unprocessed_keys`. @@ -19,51 +17,41 @@ class BatchGetItemIterator(Iterator): :: - >>> for item in BatchGetItemIterator(connection, table, {"h": 0}, {"h": 1}, {"h": 2}): + >>> for item in iterate_batch_get_item(connection, table, {"h": 0}, {"h": 1}, {"h": 2}): ... print item {u'h': 1, u'gr': 0, u'gh': 0} {u'h': 2, u'gr': 0, u'gh': 0} {u'h': 0, u'gr': 0, u'gh': 0} Note that items are returned in an unspecified order. - - A :class:`BatchGetItemIterator` instance is iterable once and must be discarded after that. """ - - @variadic(dict) - def __init__(self, connection, table, keys): - """FOO! the @variadic decorator adds doc here but we want it in the class' doc""" - self.__table = table - self.__keys = keys - self.__unprocessed_keys = [] - Iterator.__init__(self, connection, self.__next_action()) - - def __next_action(self): - if len(self.__keys) > 0: - action = _lv.BatchGetItem().table(self.__table).keys(self.__keys[:100]) - self.__keys = self.__keys[100:] - return action - elif len(self.__unprocessed_keys) > 0: - action = _lv.BatchGetItem({self.__table: {"Keys": self.__unprocessed_keys[:100]}}) - self.__unprocessed_keys = self.__unprocessed_keys[100:] - return action - else: - return None - - def process(self, action, r): - if isinstance(r.unprocessed_keys, dict) and self.__table in r.unprocessed_keys and "Keys" in r.unprocessed_keys[self.__table]: - self.__unprocessed_keys += r.unprocessed_keys[self.__table]["Keys"] - return self.__next_action(), r.responses.get(self.__table, []) - - -class BatchGetItemIteratorUnitTests(_tst.UnitTestsWithMocks): + unprocessed_keys = [] + + while len(keys) != 0: + r = connection(_lv.BatchGetItem().table(table).keys(keys[:100])) + keys = keys[100:] + if isinstance(r.unprocessed_keys, dict) and table in r.unprocessed_keys and "Keys" in r.unprocessed_keys[table]: + unprocessed_keys += r.unprocessed_keys[table]["Keys"] + for item in r.responses.get(table, []): + yield item + + while len(unprocessed_keys) != 0: + r = connection(_lv.BatchGetItem({table: {"Keys": unprocessed_keys[:100]}})) + unprocessed_keys = unprocessed_keys[100:] + if isinstance(r.unprocessed_keys, dict) and table in r.unprocessed_keys and "Keys" in r.unprocessed_keys[table]: + unprocessed_keys += r.unprocessed_keys[table]["Keys"] + for item in r.responses.get(table, []): + yield item + + +class IterateBatchGetItemUnitTests(_tst.UnitTestsWithMocks): def setUp(self): - super(BatchGetItemIteratorUnitTests, self).setUp() + super(IterateBatchGetItemUnitTests, self).setUp() self.connection = self.mocks.create("connection") def test_no_keys(self): self.assertEqual( - list(_lv.BatchGetItemIterator(self.connection.object, "Aaa", [])), + list(_lv.iterate_batch_get_item(self.connection.object, "Aaa", [])), [] ) @@ -75,7 +63,7 @@ def test_one_page(self): ) self.assertEqual( - list(_lv.BatchGetItemIterator(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"})), + list(_lv.iterate_batch_get_item(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"})), [{"h": "c"}, {"h": "d"}] ) @@ -92,7 +80,7 @@ def test_one_unprocessed_key(self): ) self.assertEqual( - list(_lv.BatchGetItemIterator(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"})), + list(_lv.iterate_batch_get_item(self.connection.object, "Aaa", {"h": u"a"}, {"h": u"b"})), [{"h": "c"}, {"h": "e"}] ) @@ -114,7 +102,7 @@ def test_several_pages(self): ) self.assertEqual( - list(_lv.BatchGetItemIterator(self.connection.object, "Aaa", ({"h": i} for i in range(0, 250)))), + list(_lv.iterate_batch_get_item(self.connection.object, "Aaa", ({"h": i} for i in range(0, 250)))), [{"h": i} for i in range(1000, 1250)] ) @@ -152,19 +140,19 @@ def test_several_pages_of_unprocessed_keys(self): ) self.assertEqual( - list(_lv.BatchGetItemIterator(self.connection.object, "Aaa", [{"h": i} for i in range(0, 150)])), + list(_lv.iterate_batch_get_item(self.connection.object, "Aaa", [{"h": i} for i in range(0, 150)])), [{"h": i} for i in range(1000, 1250)] ) -class BatchGetItemIteratorLocalIntegTests(_tst.LocalIntegTestsWithTableH): +class IterateBatchGetItemLocalIntegTests(_tst.LocalIntegTestsWithTableH): def key(self, i): return u"{:03}".format(i) def setUp(self): - super(BatchGetItemIteratorLocalIntegTests, self).setUp() - _lv.BatchPutItem(self.connection, "Aaa", [{"h": self.key(i), "xs": "x" * 300000} for i in range(250)]) # 300kB items ensure a single BatchGetItem will return at most 55 items + super(IterateBatchGetItemLocalIntegTests, self).setUp() + _lv.batch_put_item(self.connection, "Aaa", [{"h": self.key(i), "xs": "x" * 300000} for i in range(250)]) # 300kB items ensure a single BatchGetItem will return at most 55 items def test(self): - keys = [item["h"] for item in _lv.BatchGetItemIterator(self.connection, "Aaa", ({"h": self.key(i)} for i in range(250)))] + keys = [item["h"] for item in _lv.iterate_batch_get_item(self.connection, "Aaa", ({"h": self.key(i)} for i in range(250)))] self.assertEqual(sorted(keys), [self.key(i) for i in range(250)]) diff --git a/LowVoltage/compounds/list_tables_iterator.py b/LowVoltage/compounds/iterate_list_tables.py similarity index 68% rename from LowVoltage/compounds/list_tables_iterator.py rename to LowVoltage/compounds/iterate_list_tables.py index c0220b4..4c177ae 100644 --- a/LowVoltage/compounds/list_tables_iterator.py +++ b/LowVoltage/compounds/iterate_list_tables.py @@ -4,43 +4,38 @@ import LowVoltage as _lv import LowVoltage.testing as _tst -from .iterator import Iterator -class ListTablesIterator(Iterator): +def iterate_list_tables(connection): """ Make as many :class:`.ListTables` actions as needed to iterate over all tables. That is until :attr:`.ListTablesResponse.last_evaluated_table_name` is ``None``. - >>> for table in ListTablesIterator(connection): + >>> for table in iterate_list_tables(connection): ... print table LowVoltage.Tests.Doc.1 LowVoltage.Tests.Doc.2 - - A :class:`ListTablesIterator` instance is iterable once and must be discarded after that. """ - def __init__(self, connection): - Iterator.__init__(self, connection, _lv.ListTables()) - - def process(self, action, r): - if r.last_evaluated_table_name is None: - action = None - else: - action.exclusive_start_table_name(r.last_evaluated_table_name) - return action, r.table_names + r = connection(_lv.ListTables()) + for table_name in r.table_names: + yield table_name + while r.last_evaluated_table_name is not None: + r = connection(_lv.ListTables().exclusive_start_table_name(r.last_evaluated_table_name)) + for table_name in r.table_names: + yield table_name -class ListTablesIteratorUnitTests(_tst.UnitTestsWithMocks): +class IterateListTablesUnitTests(_tst.UnitTestsWithMocks): def setUp(self): - super(ListTablesIteratorUnitTests, self).setUp() + super(IterateListTablesUnitTests, self).setUp() self.connection = self.mocks.create("connection") def test_no_tables(self): self.connection.expect._call_.withArguments(self.ActionChecker("ListTables", {})).andReturn(_lv.ListTablesResponse(TableNames=[])) self.assertEqual( - list(ListTablesIterator(self.connection.object)), + list(iterate_list_tables(self.connection.object)), [] ) @@ -48,7 +43,7 @@ def test_one_page(self): self.connection.expect._call_.withArguments(self.ActionChecker("ListTables", {})).andReturn(_lv.ListTablesResponse(TableNames=["A", "B", "C"])) self.assertEqual( - list(ListTablesIterator(self.connection.object)), + list(iterate_list_tables(self.connection.object)), ["A", "B", "C"] ) @@ -57,7 +52,7 @@ def test_one_page_followed_by_empty_page(self): self.connection.expect._call_.withArguments(self.ActionChecker("ListTables", {"ExclusiveStartTableName": "D"})).andReturn(_lv.ListTablesResponse(TableNames=[])) self.assertEqual( - list(ListTablesIterator(self.connection.object)), + list(iterate_list_tables(self.connection.object)), ["A", "B", "C"] ) @@ -67,16 +62,16 @@ def test_several_pages(self): self.connection.expect._call_.withArguments(self.ActionChecker("ListTables", {"ExclusiveStartTableName": "H"})).andReturn(_lv.ListTablesResponse(TableNames=["I", "J", "K"])) self.assertEqual( - list(ListTablesIterator(self.connection.object)), + list(iterate_list_tables(self.connection.object)), ["A", "B", "C", "E", "F", "G", "I", "J", "K"] ) -class ListTablesIteratorLocalIntegTests(_tst.LocalIntegTests): +class IterateListTablesLocalIntegTests(_tst.LocalIntegTests): table_names = ["Tab{:03}".format(i) for i in range(103)] def setUp(self): - super(ListTablesIteratorLocalIntegTests, self).setUp() + super(IterateListTablesLocalIntegTests, self).setUp() for t in self.table_names: self.connection( _lv.CreateTable(t).hash_key("h", _lv.STRING).provisioned_throughput(1, 1) @@ -85,10 +80,10 @@ def setUp(self): def tearDown(self): for t in self.table_names: self.connection(_lv.DeleteTable(t)) - super(ListTablesIteratorLocalIntegTests, self).tearDown() + super(IterateListTablesLocalIntegTests, self).tearDown() def test(self): self.assertEqual( - list(_lv.ListTablesIterator(self.connection)), + list(_lv.iterate_list_tables(self.connection)), self.table_names ) diff --git a/LowVoltage/compounds/query_iterator.py b/LowVoltage/compounds/iterate_query.py similarity index 60% rename from LowVoltage/compounds/query_iterator.py rename to LowVoltage/compounds/iterate_query.py index 875a7f9..38d0c06 100644 --- a/LowVoltage/compounds/query_iterator.py +++ b/LowVoltage/compounds/iterate_query.py @@ -4,35 +4,30 @@ import LowVoltage as _lv import LowVoltage.testing as _tst -from .iterator import Iterator -class QueryIterator(Iterator): +def iterate_query(connection, query): """ Make as many :class:`.Query` actions as needed to iterate over all matching items. That is until :attr:`.QueryResponse.last_evaluated_key` is ``None``. - >>> for item in QueryIterator(connection, Query(table2).key_eq("h", 42).key_between("r1", 4, 7)): + >>> for item in iterate_query(connection, Query(table2).key_eq("h", 42).key_between("r1", 4, 7)): ... print item {u'h': 42, u'r1': 4, u'r2': 6} {u'h': 42, u'r1': 5, u'r2': 5} {u'h': 42, u'r1': 6} {u'h': 42, u'r1': 7} - A :class:`QueryIterator` instance is iterable once and must be discarded after that. - - The :class:`.Query` instance passed in must also be discarded (it is modified during the iteration). + The :class:`.Query` instance passed in must be discarded (it is modified during the iteration). """ - - def __init__(self, connection, query): - Iterator.__init__(self, connection, query) - - def process(self, action, r): - if r.last_evaluated_key is None: - action = None - else: - action.exclusive_start_key(r.last_evaluated_key) - return action, r.items + r = connection(query) + for item in r.items: + yield item + while r.last_evaluated_key is not None: + query.exclusive_start_key(r.last_evaluated_key) + r = connection(query) + for item in r.items: + yield item class QueryIteratorLocalIntegTests(_tst.LocalIntegTestsWithTableHR): @@ -49,6 +44,6 @@ def setUp(self): def test_simple_query(self): self.assertEqual( - sorted(item["r"] for item in _lv.QueryIterator(self.connection, _lv.Query("Aaa").key_eq("h", u"0"))), + sorted(item["r"] for item in _lv.iterate_query(self.connection, _lv.Query("Aaa").key_eq("h", u"0"))), self.keys ) diff --git a/LowVoltage/compounds/iterate_scan.py b/LowVoltage/compounds/iterate_scan.py new file mode 100644 index 0000000..718f692 --- /dev/null +++ b/LowVoltage/compounds/iterate_scan.py @@ -0,0 +1,95 @@ +# coding: utf8 + +# Copyright 2014-2015 Vincent Jacques + +import copy + +import LowVoltage as _lv +import LowVoltage.testing as _tst + + +def iterate_scan(connection, scan): + """ + Make as many :class:`.Scan` actions as needed to iterate over all matching items. + That is until :attr:`.ScanResponse.last_evaluated_key` is ``None``. + + >>> for item in iterate_scan(connection, Scan(table)): + ... print item + {u'h': 7, u'gr': 0, u'gh': 0} + {u'h': 8, u'gr': 0, u'gh': 0} + {u'h': 3, u'gr': 0, u'gh': 0} + {u'h': 2, u'gr': 0, u'gh': 0} + {u'h': 9, u'gr': 0, u'gh': 0} + {u'h': 4, u'gr': 0, u'gh': 0} + {u'h': 6, u'gr': 0, u'gh': 0} + {u'h': 1, u'gr': 0, u'gh': 0} + {u'h': 0, u'gr': 0, u'gh': 0} + {u'h': 5, u'gr': 0, u'gh': 0} + + The :class:`.Scan` instance passed in must be discarded (it is modified during the iteration). + """ + r = connection(scan) + for item in r.items: + yield item + while r.last_evaluated_key is not None: + scan.exclusive_start_key(r.last_evaluated_key) + r = connection(scan) + for item in r.items: + yield item + + +def parallelize_scan(scan, total_segments): + """ + Create ``total_segments`` :class:`.Scan` to be used in `parallel `__. + + >>> segments = parallelize_scan(Scan(table), 3) + + (You would typically iterate other each segment in a different thread.) + + >>> for segment in segments: + ... print "Segment" + ... for item in iterate_scan(connection, segment): + ... print item + Segment + {u'h': 7, u'gr': 0, u'gh': 0} + {u'h': 8, u'gr': 0, u'gh': 0} + {u'h': 3, u'gr': 0, u'gh': 0} + Segment + {u'h': 2, u'gr': 0, u'gh': 0} + {u'h': 9, u'gr': 0, u'gh': 0} + {u'h': 4, u'gr': 0, u'gh': 0} + {u'h': 6, u'gr': 0, u'gh': 0} + Segment + {u'h': 1, u'gr': 0, u'gh': 0} + {u'h': 0, u'gr': 0, u'gh': 0} + {u'h': 5, u'gr': 0, u'gh': 0} + """ + return [ + copy.deepcopy(scan).segment(i, total_segments) + for i in range(total_segments) + ] + + +class ScanIteratorLocalIntegTests(_tst.LocalIntegTestsWithTableH): + keys = [u"{:03}".format(k) for k in range(15)] + + def setUp(self): + super(ScanIteratorLocalIntegTests, self).setUp() + self.connection( + _lv.BatchWriteItem().table("Aaa").put( + {"h": h, "xs": "x" * 300000} # 300kB items ensure a single Query will return at most 4 items + for h in self.keys + ) + ) + + def test_simple_scan(self): + self.assertEqual( + sorted(item["h"] for item in _lv.iterate_scan(self.connection, _lv.Scan("Aaa"))), + self.keys + ) + + def test_parallel_scan(self): + keys = [] + for segment in parallelize_scan(_lv.Scan("Aaa"), 3): + keys.extend(item["h"] for item in iterate_scan(self.connection, segment)) + self.assertEqual(sorted(keys), self.keys) diff --git a/LowVoltage/compounds/iterator.py b/LowVoltage/compounds/iterator.py deleted file mode 100644 index f018d67..0000000 --- a/LowVoltage/compounds/iterator.py +++ /dev/null @@ -1,31 +0,0 @@ -# coding: utf8 - -# Copyright 2014-2015 Vincent Jacques - - -class Iterator(object): - # Don't implement anything else than a single forward iteration. - # Remember PyGithub's PaginatedList; it was too difficult to maintain for niche use-cases. - # Clients can use raw actions to implement their specific needs. - - def __init__(self, connection, first_action): - self.__connection = connection - self.__current_iter = [].__iter__() - self.__next_action = first_action - - def __iter__(self): - # @todo It seems that __iter__ can yield directly so we may be able to remove "next". - return self - - def next(self): - try: - return self.__current_iter.next() - except StopIteration: - if self.__next_action is None: - raise - else: - r = self.__connection(self.__next_action) - next_action, new_items = self.process(self.__next_action, r) - self.__next_action = next_action - self.__current_iter = new_items.__iter__() - return self.__current_iter.next() diff --git a/LowVoltage/compounds/scan_iterator.py b/LowVoltage/compounds/scan_iterator.py deleted file mode 100644 index fd9fc2d..0000000 --- a/LowVoltage/compounds/scan_iterator.py +++ /dev/null @@ -1,103 +0,0 @@ -# coding: utf8 - -# Copyright 2014-2015 Vincent Jacques - -import copy - -import LowVoltage as _lv -import LowVoltage.testing as _tst -from .iterator import Iterator - - -class ScanIterator(Iterator): - """ - Make as many :class:`.Scan` actions as needed to iterate over all matching items. - That is until :attr:`.ScanResponse.last_evaluated_key` is ``None``. - - >>> for item in ScanIterator(connection, Scan(table)): - ... print item - {u'h': 7, u'gr': 0, u'gh': 0} - {u'h': 8, u'gr': 0, u'gh': 0} - {u'h': 3, u'gr': 0, u'gh': 0} - {u'h': 2, u'gr': 0, u'gh': 0} - {u'h': 9, u'gr': 0, u'gh': 0} - {u'h': 4, u'gr': 0, u'gh': 0} - {u'h': 6, u'gr': 0, u'gh': 0} - {u'h': 1, u'gr': 0, u'gh': 0} - {u'h': 0, u'gr': 0, u'gh': 0} - {u'h': 5, u'gr': 0, u'gh': 0} - - A :class:`ScanIterator` instance is iterable once and must be discarded after that. - - The :class:`.Scan` instance passed in must also be discarded (it is modified during the iteration). - """ - - @classmethod - def parallelize(cls, connection, scan, total_segments): - """ - Create ``total_segments`` :class:`ScanIterator` to be used in `parallel `__. - - >>> segments = ScanIterator.parallelize(connection, Scan(table), 3) - - (You would typically iterate other each segment in a different thread.) - - >>> for segment in segments: - ... print "Segment" - ... for item in segment: - ... print item - Segment - {u'h': 7, u'gr': 0, u'gh': 0} - {u'h': 8, u'gr': 0, u'gh': 0} - {u'h': 3, u'gr': 0, u'gh': 0} - Segment - {u'h': 2, u'gr': 0, u'gh': 0} - {u'h': 9, u'gr': 0, u'gh': 0} - {u'h': 4, u'gr': 0, u'gh': 0} - {u'h': 6, u'gr': 0, u'gh': 0} - Segment - {u'h': 1, u'gr': 0, u'gh': 0} - {u'h': 0, u'gr': 0, u'gh': 0} - {u'h': 5, u'gr': 0, u'gh': 0} - - The :class:`.Scan` instance passed in must be discarded (it is modified during the iteration). - """ - # The scan instance is actually preserved but this is an implementation detail that should not be relied on. - return [ - cls(connection, copy.deepcopy(scan).segment(i, total_segments)) - for i in range(total_segments) - ] - - def __init__(self, connection, scan): - Iterator.__init__(self, connection, scan) - - def process(self, action, r): - if r.last_evaluated_key is None: - action = None - else: - action.exclusive_start_key(r.last_evaluated_key) - return action, r.items - - -class ScanIteratorLocalIntegTests(_tst.LocalIntegTestsWithTableH): - keys = [u"{:03}".format(k) for k in range(15)] - - def setUp(self): - super(ScanIteratorLocalIntegTests, self).setUp() - self.connection( - _lv.BatchWriteItem().table("Aaa").put( - {"h": h, "xs": "x" * 300000} # 300kB items ensure a single Query will return at most 4 items - for h in self.keys - ) - ) - - def test_simple_scan(self): - self.assertEqual( - sorted(item["h"] for item in _lv.ScanIterator(self.connection, _lv.Scan("Aaa"))), - self.keys - ) - - def test_parallel_scan(self): - keys = [] - for segment in ScanIterator.parallelize(self.connection, _lv.Scan("Aaa"), 3): - keys.extend(item["h"] for item in segment) - self.assertEqual(sorted(keys), self.keys) diff --git a/LowVoltage/compounds/tests/integ/local.py b/LowVoltage/compounds/tests/integ/local.py index 69ac502..56c49b5 100644 --- a/LowVoltage/compounds/tests/integ/local.py +++ b/LowVoltage/compounds/tests/integ/local.py @@ -3,10 +3,10 @@ # Copyright 2014-2015 Vincent Jacques from ...batch_delete_item import BatchDeleteItemLocalIntegTests -from ...batch_get_item_iterator import BatchGetItemIteratorLocalIntegTests +from ...iterate_batch_get_item import IterateBatchGetItemLocalIntegTests from ...batch_put_item import BatchPutItemLocalIntegTests -from ...list_tables_iterator import ListTablesIteratorLocalIntegTests -from ...query_iterator import QueryIteratorLocalIntegTests -from ...scan_iterator import ScanIteratorLocalIntegTests +from ...iterate_list_tables import IterateListTablesLocalIntegTests +from ...iterate_query import QueryIteratorLocalIntegTests +from ...iterate_scan import ScanIteratorLocalIntegTests from ...wait_for_table_activation import WaitForTableActivationLocalIntegTests from ...wait_for_table_deletion import WaitForTableDeletionLocalIntegTests diff --git a/LowVoltage/compounds/tests/unit.py b/LowVoltage/compounds/tests/unit.py index 90386aa..40562ff 100644 --- a/LowVoltage/compounds/tests/unit.py +++ b/LowVoltage/compounds/tests/unit.py @@ -2,7 +2,7 @@ # Copyright 2014-2015 Vincent Jacques -from ..list_tables_iterator import ListTablesIteratorUnitTests -from ..batch_get_item_iterator import BatchGetItemIteratorUnitTests +from ..iterate_list_tables import IterateListTablesUnitTests +from ..iterate_batch_get_item import IterateBatchGetItemUnitTests from ..batch_delete_item import BatchDeleteItemUnitTests from ..batch_put_item import BatchPutItemUnitTests diff --git a/LowVoltage/compounds/wait_for_table_activation.py b/LowVoltage/compounds/wait_for_table_activation.py index 2afcda5..15e52cc 100644 --- a/LowVoltage/compounds/wait_for_table_activation.py +++ b/LowVoltage/compounds/wait_for_table_activation.py @@ -8,7 +8,7 @@ import LowVoltage.testing as _tst -def WaitForTableActivation(connection, table): +def wait_for_table_activation(connection, table): """ Make :class:`.DescribeTable` actions until the table's status (and all its GSI's statuses) is ```"ACTIVE"```. Useful after :class:`.CreateTable` and :class:`.UpdateTable` actions. @@ -23,14 +23,14 @@ def WaitForTableActivation(connection, table): ... .provisioned_throughput(1, 1) ... ).table_description.table_status u'CREATING' - >>> WaitForTableActivation(connection, table) + >>> wait_for_table_activation(connection, table) >>> connection(DescribeTable(table)).table.table_status u'ACTIVE' .. testcleanup:: connection(DeleteTable(table)) - WaitForTableDeletion(connection, table) + wait_for_table_deletion(connection, table) """ r = connection(_lv.DescribeTable(table)) @@ -46,7 +46,7 @@ def tearDown(self): def test(self): self.connection(_lv.CreateTable("Aaa").hash_key("h", _lv.STRING).provisioned_throughput(1, 1)) - _lv.WaitForTableActivation(self.connection, "Aaa") + _lv.wait_for_table_activation(self.connection, "Aaa") self.assertEqual(self.connection(_lv.DescribeTable("Aaa")).table.table_status, "ACTIVE") @@ -64,7 +64,7 @@ def test(self): _lv.CreateTable(self.table).hash_key("tab_h", _lv.STRING).range_key("tab_r", _lv.NUMBER).provisioned_throughput(1, 1) .global_secondary_index("gsi").hash_key("gsi_h", _lv.STRING).range_key("gsi_r", _lv.NUMBER).project_all().provisioned_throughput(1, 1) ) - _lv.WaitForTableActivation(self.connection, self.table) + _lv.wait_for_table_activation(self.connection, self.table) r = self.connection(_lv.DescribeTable(self.table)) self.assertEqual(r.table.table_status, "ACTIVE") self.assertEqual(r.table.global_secondary_indexes[0].index_status, "ACTIVE") diff --git a/LowVoltage/compounds/wait_for_table_deletion.py b/LowVoltage/compounds/wait_for_table_deletion.py index 7da889b..d2a7d06 100644 --- a/LowVoltage/compounds/wait_for_table_deletion.py +++ b/LowVoltage/compounds/wait_for_table_deletion.py @@ -9,7 +9,7 @@ # @todo Should we abort if the table_status is not "DELETING"? Better provide a DeleteTableAndWait compound -def WaitForTableDeletion(connection, table): +def wait_for_table_deletion(connection, table): """ Make :class:`.DescribeTable` actions until a :exc:`.ResourceNotFoundException` is raised. Useful after :class:`.DeleteTable` action. @@ -18,11 +18,11 @@ def WaitForTableDeletion(connection, table): table = "LowVoltage.Tests.Doc.WaitForTableDeletion.1" connection(CreateTable(table).hash_key("h", STRING).provisioned_throughput(1, 1)) - WaitForTableActivation(connection, table) + wait_for_table_activation(connection, table) >>> connection(DeleteTable(table)).table_description.table_status u'DELETING' - >>> WaitForTableDeletion(connection, table) + >>> wait_for_table_deletion(connection, table) >>> connection(DescribeTable(table)) Traceback (most recent call last): ... @@ -44,7 +44,7 @@ def setUp(self): def test(self): self.connection(_lv.DeleteTable("Aaa")) - _lv.WaitForTableDeletion(self.connection, "Aaa") + _lv.wait_for_table_deletion(self.connection, "Aaa") with self.assertRaises(_lv.ResourceNotFoundException): self.connection(_lv.DescribeTable("Aaa")) @@ -54,10 +54,10 @@ def setUp(self): super(WaitForTableDeletionConnectedIntegTests, self).setUp() self.table = self.make_table_name() self.connection(_lv.CreateTable(self.table).hash_key("h", _lv.STRING).provisioned_throughput(1, 1)) - _lv.WaitForTableActivation(self.connection, self.table) + _lv.wait_for_table_activation(self.connection, self.table) def test(self): self.connection(_lv.DeleteTable(self.table)) - _lv.WaitForTableDeletion(self.connection, self.table) + _lv.wait_for_table_deletion(self.connection, self.table) with self.assertRaises(_lv.ResourceNotFoundException): self.connection(_lv.DescribeTable(self.table)) diff --git a/LowVoltage/testing/connected_integ_tests.py b/LowVoltage/testing/connected_integ_tests.py index fcded2b..776c6b0 100644 --- a/LowVoltage/testing/connected_integ_tests.py +++ b/LowVoltage/testing/connected_integ_tests.py @@ -68,14 +68,14 @@ def make(self, dependencies): .global_secondary_index("gsi").hash_key("gsi_h", _lv.STRING).range_key("gsi_r", _lv.NUMBER).project_all().provisioned_throughput(1, 1) .local_secondary_index("lsi").hash_key("tab_h").range_key("lsi_r", _lv.NUMBER).project_all().provisioned_throughput(1, 1) ) - _lv.WaitForTableActivation(connection, table) + _lv.wait_for_table_activation(connection, table) return table def clean(self, table): connection = make_connection() connection(_lv.DeleteTable(table)) - _lv.WaitForTableDeletion(connection, table) + _lv.wait_for_table_deletion(connection, table) class ConnectedIntegTestsWithTable(ConnectedIntegTests): diff --git a/LowVoltage/testing/doc_tests.py b/LowVoltage/testing/doc_tests.py index bb1ad2e..64c5df0 100644 --- a/LowVoltage/testing/doc_tests.py +++ b/LowVoltage/testing/doc_tests.py @@ -29,27 +29,27 @@ def global_setup(): .local_secondary_index("lsi").hash_key("h", _lv.NUMBER).range_key("r2", _lv.NUMBER).project_all() ) - _lv.WaitForTableActivation(connection, table1) - _lv.BatchPutItem( + _lv.wait_for_table_activation(connection, table1) + _lv.batch_put_item( connection, table1, [{"h": h, "gh": 0, "gr": 0} for h in range(10)], ) - _lv.WaitForTableActivation(connection, table2) - _lv.BatchPutItem( + _lv.wait_for_table_activation(connection, table2) + _lv.batch_put_item( connection, table2, [{"h": h, "r1": 0, "r2": 0} for h in range(10)], ) - _lv.BatchPutItem( + _lv.batch_put_item( connection, table2, [{"h": 42, "r1": r1, "r2": 10 - r1} for r1 in range(6)], ) - _lv.BatchPutItem( + _lv.batch_put_item( connection, table2, [{"h": 42, "r1": r1} for r1 in range(6, 10)], diff --git a/doc/changelog.rst b/doc/changelog.rst index e2dab97..ef546ad 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -2,7 +2,12 @@ Changelog ========= +????/??/?? v0.7.0 +================== + +- A few hours after writing "I believe the interface is stable", I had an aha moment and had to change the :ref:`compounds`, making them all functions instead of a mix of classes and badly named functions. + 2015/04/26, v0.6.0 ================== -This is the first beta version, so the first to appear in the changelog. +- This is the first beta version, so the first to appear in the changelog. diff --git a/doc/reference.rst b/doc/reference.rst index 1710b3e..6c96315 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -97,11 +97,11 @@ See also :ref:`actions-vs-compounds` in the user guide. .. toctree:: - reference/compounds/batch_get_item_iterator + reference/compounds/iterate_batch_get_item reference/compounds/batch_put_item reference/compounds/batch_delete_item - reference/compounds/list_tables_iterator - reference/compounds/scan_iterator - reference/compounds/query_iterator + reference/compounds/iterate_list_tables + reference/compounds/iterate_scan + reference/compounds/iterate_query reference/compounds/wait_for_table_activation reference/compounds/wait_for_table_deletion diff --git a/doc/reference/compounds/batch_delete_item.rst b/doc/reference/compounds/batch_delete_item.rst index 1fa1566..b670b4c 100644 --- a/doc/reference/compounds/batch_delete_item.rst +++ b/doc/reference/compounds/batch_delete_item.rst @@ -1,4 +1,4 @@ -BatchDeleteItem -=============== +batch_delete_item +================= .. automodule:: LowVoltage.compounds.batch_delete_item diff --git a/doc/reference/compounds/batch_get_item_iterator.rst b/doc/reference/compounds/batch_get_item_iterator.rst deleted file mode 100644 index 2260bc4..0000000 --- a/doc/reference/compounds/batch_get_item_iterator.rst +++ /dev/null @@ -1,4 +0,0 @@ -BatchGetItemIterator -==================== - -.. automodule:: LowVoltage.compounds.batch_get_item_iterator diff --git a/doc/reference/compounds/batch_put_item.rst b/doc/reference/compounds/batch_put_item.rst index 9dc9b00..c40a70c 100644 --- a/doc/reference/compounds/batch_put_item.rst +++ b/doc/reference/compounds/batch_put_item.rst @@ -1,4 +1,4 @@ -BatchPutItem -============ +batch_put_item +============== .. automodule:: LowVoltage.compounds.batch_put_item diff --git a/doc/reference/compounds/iterate_batch_get_item.rst b/doc/reference/compounds/iterate_batch_get_item.rst new file mode 100644 index 0000000..ac26d2e --- /dev/null +++ b/doc/reference/compounds/iterate_batch_get_item.rst @@ -0,0 +1,4 @@ +iterate_batch_get_item +====================== + +.. automodule:: LowVoltage.compounds.iterate_batch_get_item diff --git a/doc/reference/compounds/iterate_list_tables.rst b/doc/reference/compounds/iterate_list_tables.rst new file mode 100644 index 0000000..445bd0d --- /dev/null +++ b/doc/reference/compounds/iterate_list_tables.rst @@ -0,0 +1,4 @@ +iterate_list_tables +=================== + +.. automodule:: LowVoltage.compounds.iterate_list_tables diff --git a/doc/reference/compounds/iterate_query.rst b/doc/reference/compounds/iterate_query.rst new file mode 100644 index 0000000..bc40889 --- /dev/null +++ b/doc/reference/compounds/iterate_query.rst @@ -0,0 +1,4 @@ +iterate_query +============= + +.. automodule:: LowVoltage.compounds.iterate_query diff --git a/doc/reference/compounds/iterate_scan.rst b/doc/reference/compounds/iterate_scan.rst new file mode 100644 index 0000000..d55a6fd --- /dev/null +++ b/doc/reference/compounds/iterate_scan.rst @@ -0,0 +1,4 @@ +iterate_scan +============ + +.. automodule:: LowVoltage.compounds.iterate_scan diff --git a/doc/reference/compounds/list_tables_iterator.rst b/doc/reference/compounds/list_tables_iterator.rst deleted file mode 100644 index fcaf2d8..0000000 --- a/doc/reference/compounds/list_tables_iterator.rst +++ /dev/null @@ -1,4 +0,0 @@ -ListTablesIterator -================== - -.. automodule:: LowVoltage.compounds.list_tables_iterator diff --git a/doc/reference/compounds/query_iterator.rst b/doc/reference/compounds/query_iterator.rst deleted file mode 100644 index eb359c1..0000000 --- a/doc/reference/compounds/query_iterator.rst +++ /dev/null @@ -1,4 +0,0 @@ -QueryIterator -============= - -.. automodule:: LowVoltage.compounds.query_iterator diff --git a/doc/reference/compounds/scan_iterator.rst b/doc/reference/compounds/scan_iterator.rst deleted file mode 100644 index 7db3657..0000000 --- a/doc/reference/compounds/scan_iterator.rst +++ /dev/null @@ -1,4 +0,0 @@ -ScanIterator -============ - -.. automodule:: LowVoltage.compounds.scan_iterator diff --git a/doc/reference/compounds/wait_for_table_activation.rst b/doc/reference/compounds/wait_for_table_activation.rst index 8d6f053..183c5f7 100644 --- a/doc/reference/compounds/wait_for_table_activation.rst +++ b/doc/reference/compounds/wait_for_table_activation.rst @@ -1,4 +1,4 @@ -WaitForTableActivation -====================== +wait_for_table_activation +========================= .. automodule:: LowVoltage.compounds.wait_for_table_activation diff --git a/doc/reference/compounds/wait_for_table_deletion.rst b/doc/reference/compounds/wait_for_table_deletion.rst index 4a830da..e840eb1 100644 --- a/doc/reference/compounds/wait_for_table_deletion.rst +++ b/doc/reference/compounds/wait_for_table_deletion.rst @@ -1,4 +1,4 @@ -WaitForTableDeletion -==================== +wait_for_table_deletion +======================= .. automodule:: LowVoltage.compounds.wait_for_table_deletion diff --git a/doc/user_guide.rst b/doc/user_guide.rst index 97b2d52..288060a 100644 --- a/doc/user_guide.rst +++ b/doc/user_guide.rst @@ -88,14 +88,14 @@ It does convert between :ref:`python-types` and DynamoDB notation though. >>> connection(GetItem(table, {"h": 0})).item {u'h': 0, u'gr': 0, u'gh': 0} -The :ref:`compounds` layer provides helper that intend to complete actions in their simplest use cases. -For example :class:`.BatchGetItem` is limited to get 100 keys at once and requires processing :attr:`.BatchGetItemResponse.unprocessed_keys`, so we provide :class:`.BatchGetItemIterator` to do that. +The :ref:`compounds` layer provides helper functions that intend to complete actions in their simplest use cases. +For example :class:`.BatchGetItem` is limited to get 100 keys at once and requires processing :attr:`.BatchGetItemResponse.unprocessed_keys`, so we provide :func:`.iterate_batch_get_item` to do that. The tradeoff is that you loose :attr:`.BatchGetItemResponse.consumed_capacity` and the ability to get items from several tables at once. -Similarly :func:`.BatchPutItem` remove the limit of 25 items in :class:`.BatchWriteItem` but also removes the ability to put and delete from several tables in the same action. +Similarly :func:`.batch_put_item` removes the limit of 25 items in :class:`.BatchWriteItem` but also removes the ability to put and delete from several tables in the same action. - >>> BatchPutItem(connection, table, {"h": 0, "a": 42}, {"h": 1, "b": 53}) + >>> batch_put_item(connection, table, {"h": 0, "a": 42}, {"h": 1, "b": 53}) -You can easily distinguish between actions and compounds because actions are *passed to* the :class:`.Connection` but compounds *receive* the connection as an argument: +Actions are instances that are *passed to* the :class:`.Connection` but compounds are functions that *receive* the connection as an argument: actions are atomic while compounds are able to perform several actions. Someday, maybe, we'll write a Table abstraction and implement an "active record" pattern? It would be even simpler than compounds, but less flexible. diff --git a/setup.py b/setup.py index d296e1b..6fe9d60 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import os import setuptools -version = "0.6.0" +version = "0.7.0" setuptools.setup(