Skip to content
Permalink
Browse files

Exhaust all iterators to avoid memory leak

az482

In the select_fun for the intersection_op (i.e. AND) it would stop
once one of it's iterators had been exhausted but would leave the
other one as is, messages and all. So in the case where one side is
empty and other side has a bunch of results (or this query is run a
lot, memory usage = result size * # of queries) the results just sit
on a process that waits indefinitely for something to pull the results
off it's mailbox.  The reason this doesn't occur if you use a PB conn
for each transaction is because he PB conn is the parent of these
processes and they are all linked.  If it dies they all die.

The solution is to make sure all iterators are exhausted.
  • Loading branch information...
rzezeski committed Jun 24, 2011
1 parent 9710475 commit cd910c2519f94e9d7e8a8e21894db9d0eecdd5b4
Showing with 13 additions and 8 deletions.
  1. +13 −8 apps/riak_search/src/riak_search_op_intersection.erl
@@ -85,10 +85,12 @@ select_fun({Term1, false, Iterator1}, {Term2, false, Iterator2}) when ?INDEX_DOC
select_fun({Term1, false, Iterator1}, {Term2, false, Iterator2}) when ?INDEX_DOCID(Term1) > ?INDEX_DOCID(Term2) ->
select_fun({Term1, false, Iterator1}, Iterator2());

select_fun({eof, false}, {_Term2, false, _Iterator2}) ->
select_fun({eof, false}, {_Term2, false, Iterator2}) ->
exhaust_it(Iterator2()),
{eof, false};

select_fun({_Term1, false, _Iterator1}, {eof, false}) ->
select_fun({_Term1, false, Iterator1}, {eof, false}) ->
exhaust_it(Iterator1()),
{eof, false};

%% Handle 'AND' cases, notflags = [false, true]
@@ -101,7 +103,8 @@ select_fun({Term1, false, Iterator1}, {Term2, true, Iterator2}) when ?INDEX_DOCI
select_fun({Term1, false, Iterator1}, {Term2, true, Iterator2}) when ?INDEX_DOCID(Term1) > ?INDEX_DOCID(Term2) ->
select_fun({Term1, false, Iterator1}, Iterator2());

select_fun({eof, false}, {_Term2, true, _Iterator2}) ->
select_fun({eof, false}, {_Term2, true, Iterator2}) ->
exhaust_it(Iterator2()),
{eof, false};

select_fun({Term1, false, Iterator1}, {eof, true}) ->
@@ -148,8 +151,10 @@ select_fun(Iterator1, Iterator2) ->
?PRINT({select_fun, unhandled_case, 'intersection', Iterator1, Iterator2}),
throw({select_fun, unhandled_case, 'intersection', Iterator1, Iterator2}).






%% @private
%%
%% @doc Exhaust the iterator of all results.
exhaust_it({eof, _}) ->
ok;
exhaust_it({_,_,It}) ->
exhaust_it(It()).

0 comments on commit cd910c2

Please sign in to comment.
You can’t perform that action at this time.