Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
Merge remote branch 'cloudant:79066-port-chunkified-replicate_batch'
This closes #26

Signed-off-by: Eric Avdey <eiri@eiri.ca>
  • Loading branch information
eiri committed Nov 24, 2016
2 parents 252467c + c4da61c commit c3c5429180de14a2b139f7741c934143ef73988c
Showing 1 changed file with 30 additions and 2 deletions.
@@ -252,8 +252,10 @@ replicate_batch(#acc{target = #shard{node=Node, name=Name}} = Acc) ->
[] ->
ok;
Missing ->
Docs = open_docs(Acc, Missing),
ok = save_on_target(Node, Name, Docs)
lists:map(fun(Chunk) ->
Docs = open_docs(Acc, Chunk),
ok = save_on_target(Node, Name, Docs)
end, chunk_revs(Missing))
end,
update_locals(Acc),
{ok, Acc#acc{revcount=0, infos=[]}}.
@@ -271,6 +273,32 @@ find_missing_revs(Acc) ->
]).


chunk_revs(Revs) ->
Limit = list_to_integer(config:get("mem3", "rev_chunk_size", "5000")),
chunk_revs(Revs, Limit).

chunk_revs(Revs, Limit) ->
chunk_revs(Revs, {0, []}, [], Limit).

chunk_revs([], {_Count, Chunk}, Chunks, _Limit) ->
[Chunk|Chunks];
chunk_revs([{Id, R, A}|Revs], {Count, Chunk}, Chunks, Limit) when length(R) =< Limit - Count ->
chunk_revs(
Revs,
{Count + length(R), [{Id, R, A}|Chunk]},
Chunks,
Limit
);
chunk_revs([{Id, R, A}|Revs], {Count, Chunk}, Chunks, Limit) ->
{This, Next} = lists:split(Limit - Count, R),
chunk_revs(
[{Id, Next, A}|Revs],
{0, []},
[[{Id, This, A}|Chunk]|Chunks],
Limit
).


open_docs(#acc{source=Source, infos=Infos}, Missing) ->
lists:flatmap(fun({Id, Revs, _}) ->
FDI = lists:keyfind(Id, #full_doc_info.id, Infos),

0 comments on commit c3c5429

Please sign in to comment.