Skip to content

Commit

Permalink
Change behavior of 'all' flag in bulk insert
Browse files Browse the repository at this point in the history
  • Loading branch information
odedlaz committed Apr 6, 2017
1 parent 7613468 commit 412f4e4
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 7 deletions.
14 changes: 9 additions & 5 deletions tests/test_bulk_utility.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,13 @@ def test_bulk_verbose_output(self):
return_value = [succeed(output)]

self.bulk_utility.streaming_bulk = MagicMock(return_value=return_value)
inserted, errors = yield self.bulk_utility.bulk(None, verbose=True)
self.assertEqual([SUCCESS] * 2, inserted)
self.assertEqual([ERROR_MSG] * 3, errors)
items = yield self.bulk_utility.bulk(None, verbose=True)

self.assertEqual([ITEM_SUCCESS] * 2,
filter(lambda x: x == ITEM_SUCCESS, items))

self.assertEqual([ITEM_FAILED] * 3,
filter(lambda x: x == ITEM_FAILED, items))

def test_streaming_bulk(self):
self.bulk_utility._process_bulk_chunk = MagicMock()
Expand All @@ -72,8 +76,8 @@ def test_streaming_bulk(self):

cb = MagicMock()

bulk = list(self.bulk_utility.streaming_bulk(range(num_of_actions),
expand_action_callback=cb))
list(self.bulk_utility.streaming_bulk(range(num_of_actions),
expand_action_callback=cb))

self.assertEqual(num_of_actions,
cb.call_count)
Expand Down
5 changes: 3 additions & 2 deletions twistes/bulk_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,16 +73,17 @@ def bulk(self, actions, stats_only=False, verbose=False, **kwargs):

inserted = []
errors = []

all = []
for deferred_bulk in self.streaming_bulk(actions, **kwargs):
bulk_results = yield deferred_bulk
for ok, item in bulk_results:
# go through request-response pairs and detect failures
all.append((ok, item))
l = inserted if ok else errors
l.append(item)

if verbose:
returnValue((inserted, errors))
returnValue(all)

if stats_only:
returnValue((len(inserted), len(errors)))
Expand Down

0 comments on commit 412f4e4

Please sign in to comment.