Skip to content

Commit

Permalink
Change behavior of 'all' flag in bulk insert (#51)
Browse files Browse the repository at this point in the history
* Change behavior of 'all' flag in bulk insert

* Update envrc file
  • Loading branch information
odedlaz authored and avihad committed Apr 13, 2017
1 parent 7613468 commit ad24a35
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 14 deletions.
17 changes: 10 additions & 7 deletions .envrc
Original file line number Diff line number Diff line change
@@ -1,17 +1,20 @@
layout python python2
if [ ! -f ".direnv/direnv.lock" ]; then
python_version=python2
lock_file="direnv.$python_version.lock"

layout python $python_version

if [ ! -f ".direnv/$lock_file" ]; then
date +%FT%TZ > ".direnv/$lock_file"

for req in requirements requirements-test; do
if [ -f $req.txt ]; then
echo "direnv: installing project $req"
pip install -r $req.txt 1> /dev/null
pip install -r $req.txt
fi
done

for package in ipython tox; do
for package in ipython pytest tox; do
echo "direnv: installing $package"
pip install --upgrade $package 1> /dev/null
pip install --upgrade $package
done

date +%FT%TZ > .direnv/direnv.lock
fi
14 changes: 9 additions & 5 deletions tests/test_bulk_utility.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,13 @@ def test_bulk_verbose_output(self):
return_value = [succeed(output)]

self.bulk_utility.streaming_bulk = MagicMock(return_value=return_value)
inserted, errors = yield self.bulk_utility.bulk(None, verbose=True)
self.assertEqual([SUCCESS] * 2, inserted)
self.assertEqual([ERROR_MSG] * 3, errors)
items = yield self.bulk_utility.bulk(None, verbose=True)

self.assertEqual([ITEM_SUCCESS] * 2,
[x for x in items if x == ITEM_SUCCESS])

self.assertEqual([ITEM_FAILED] * 3,
[x for x in items if x == ITEM_FAILED])

def test_streaming_bulk(self):
self.bulk_utility._process_bulk_chunk = MagicMock()
Expand All @@ -72,8 +76,8 @@ def test_streaming_bulk(self):

cb = MagicMock()

bulk = list(self.bulk_utility.streaming_bulk(range(num_of_actions),
expand_action_callback=cb))
list(self.bulk_utility.streaming_bulk(range(num_of_actions),
expand_action_callback=cb))

self.assertEqual(num_of_actions,
cb.call_count)
Expand Down
5 changes: 3 additions & 2 deletions twistes/bulk_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,16 +73,17 @@ def bulk(self, actions, stats_only=False, verbose=False, **kwargs):

inserted = []
errors = []

all = []
for deferred_bulk in self.streaming_bulk(actions, **kwargs):
bulk_results = yield deferred_bulk
for ok, item in bulk_results:
# go through request-response pairs and detect failures
all.append((ok, item))
l = inserted if ok else errors
l.append(item)

if verbose:
returnValue((inserted, errors))
returnValue(all)

if stats_only:
returnValue((len(inserted), len(errors)))
Expand Down

0 comments on commit ad24a35

Please sign in to comment.