Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

add `seq_indexed` option to a design document.

If the option `seq_indexed` is **true**, the views indexes will be also
indexed by sequences. It is possible to retrieve changes using the
`couch_mrview:view_changes_since/{5,6,7}`  function wich allows you to
get all changes in a view or changes from a view in a range since last
sequence. ex:

    couch_mrview:view_changes_since(Db, <<"_design/test">>, <<"test">>,
0, fun(KV, Acc) -> io:format("kv ~p~n", [KV]), {ok, Acc} end,
[{start_key, null}, {end_key, null}]).
    kv {{null,4},
        {<<"1f7d72c4ae8cab487c7efeff050011fe">>,
         {[{<<"_id">>,<<"1f7d72c4ae8cab487c7efeff050011fe">>},
           {<<"_rev">>,<<"1-9b2a5b981a9dd0ad1b3a3e826498e9bb">>},
           {<<"test">>,null}]}}}
    {ok,[]}

If the range is omitted all changes will be retrieved since the last
sequence.

Internals:

When `seq_indexed` is created 1 main sequence is created for all the
view group with the key `{ViewId, Seq}` and the value `{DocId, Key,
Val}`
is added. There can be multiple lines by sequences. These sequences are
retrieved by querying the b-tree in the range:  `[{ViewId, StartSeq+1},
{ViewId, EndSeq}]`
where EndSeq depending on the direction can be 0 or a the last big
integer.

To allows the querying of changes in a range, a new index is added per
views with the composite key `{Key, Seq}` (instead DocId) and the value
`{DocId, Val}` is indexed for this key. We query this index in the range
`[{StartKey, StartSeq+1}, {EndKey, EndSeq}]` to retrieve the changes.

Caveats:

The size of the indexes will grow significantly with this changes and
compaction is needed more often.
  • Loading branch information...
commit 14ad584c993e7a25c000d8db905fd0d0a88a24b4 1 parent d1850f2
@benoitc benoitc authored
View
5 apps/couch_mrview/include/couch_mrview.hrl
@@ -19,9 +19,11 @@
language,
design_opts=[],
include_deleted=false,
+ seq_indexed=false,
lib,
views,
id_btree=nil,
+ seq_btree=nil,
update_seq=0,
purge_seq=0,
@@ -36,12 +38,14 @@
-record(mrview, {
id_num,
+ seq_indexed=false,
update_seq=0,
purge_seq=0,
map_names=[],
reduce_funs=[],
def,
btree=nil,
+ seq_btree=nil,
options=[]
}).
@@ -50,6 +54,7 @@
seq=0,
purge_seq=0,
id_btree_state=nil,
+ seq_btree_state=nil,
view_states=nil
}).
View
67 apps/couch_mrview/src/couch_mrview.erl
@@ -14,6 +14,7 @@
-export([query_all_docs/2, query_all_docs/4]).
-export([query_view/3, query_view/4, query_view/6]).
+-export([view_changes_since/5, view_changes_since/6, view_changes_since/7]).
-export([get_info/2]).
-export([compact/2, compact/3, cancel_compaction/2]).
-export([cleanup/1]).
@@ -72,7 +73,8 @@ query_view(Db, DDoc, VName, Args) ->
query_view(Db, DDoc, VName, Args, Callback, Acc) when is_list(Args) ->
query_view(Db, DDoc, VName, to_mrargs(Args), Callback, Acc);
query_view(Db, DDoc, VName, Args0, Callback, Acc0) ->
- {ok, VInfo, Sig, Args} = couch_mrview_util:get_view(Db, DDoc, VName, Args0),
+ {ok, VInfo, _State, Sig, Args} = couch_mrview_util:get_view(Db, DDoc,
+ VName, Args0),
{ok, Acc1} = case Args#mrargs.preflight_fun of
PFFun when is_function(PFFun, 2) -> PFFun(Sig, Acc0);
_ -> {ok, Acc0}
@@ -86,6 +88,69 @@ query_view(Db, {Type, View}, Args, Callback, Acc) ->
red -> red_fold(Db, View, Args, Callback, Acc)
end.
+view_changes_since(Db, DDoc, VName, StartSeq, Callback) ->
+ view_changes_since(Db, DDoc, VName, StartSeq, Callback, #mrargs{}, []).
+
+view_changes_since(Db, DDoc, VName, StartSeq, Callback, Args) ->
+ view_changes_since(Db, DDoc, VName, StartSeq, Callback, Args, []).
+
+view_changes_since(Db, DDoc, VName, StartSeq, Callback, Args, Acc)
+ when is_list(Args) ->
+ view_changes_since(Db, DDoc, VName, StartSeq, Callback,
+ to_mrargs(Args), Acc);
+view_changes_since(Db, DDoc, VName, StartSeq, Callback,
+ #mrargs{direction=Dir}=Args0, Acc) ->
+ EndSeq = case Dir of
+ fwd -> 16#10000000;
+ rev -> 0
+ end,
+ {ok, VInfo, State, _Sig, Args} = couch_mrview_util:get_view(Db, DDoc,
+ VName, Args0),
+
+ {_Type, View} = VInfo,
+ case View#mrview.seq_indexed of
+ true ->
+ Acc0 = {Dir, StartSeq, Acc},
+ WrapperFun = fun
+ ({{_ViewId, Seq}, {DocId, Key, Val}}, _Reds, {_, _, Acc2}) ->
+ {ok, Acc3} = Callback({{Key, Seq}, {DocId, Val}}, Acc2),
+ {ok, {fwd, Seq, Acc3}};
+ ({{_Key, Seq}, _DocId}=KV, _Reds, {fwd, LastSeq, Acc2})
+ when Seq > LastSeq, Seq =< EndSeq ->
+ {ok, Acc3} = Callback(KV, Acc2),
+ {ok, {fwd, Seq, Acc3}};
+ ({{_Key, Seq}, _DocId}=KV, _Reds, {D, LastSeq, Acc2})
+ when Seq < LastSeq, Seq >= EndSeq ->
+ {ok, Acc3} = Callback(KV, Acc2),
+ {ok, {D, Seq, Acc3}};
+ (_, _, Acc2) ->
+ {ok, Acc2}
+ end,
+ {Btree, Opts} = case {Args#mrargs.start_key, Args#mrargs.end_key} of
+ {undefined, undefined} ->
+ #mrst{seq_btree=SeqBtree} = State,
+ #mrview{id_num=Id}=View,
+ {SeqBtree, [{start_key, {Id, StartSeq+1}},
+ {end_key, {Id, EndSeq}}]};
+ {SK, undefined} ->
+ {View#mrview.seq_btree, [{start_key, {SK, StartSeq+1}},
+ {end_key, {SK, EndSeq}}]};
+ {undefined, EK} ->
+ {View#mrview.seq_btree, [{start_key, {EK, StartSeq+1}},
+ {end_key, {EK, EndSeq}}]};
+ {SK, EK} ->
+ {View#mrview.seq_btree, [{start_key, {SK, StartSeq+1}},
+ {end_key, {EK, EndSeq}}]}
+ end,
+
+ {ok, _R, {_, _, AccOut}} = couch_btree:fold(Btree, WrapperFun,
+ Acc0, Opts),
+ {ok, AccOut};
+ _ ->
+ {error, seqs_not_indexed}
+ end.
+
+
get_info(Db, DDoc) ->
{ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
View
94 apps/couch_mrview/src/couch_mrview_compactor.erl
@@ -18,13 +18,12 @@
-export([compact/3, swap_compacted/2]).
-record(acc, {
- btree = nil,
- last_id = nil,
- kvs = [],
- kvs_size = 0,
- changes = 0,
- total_changes
-}).
+ btree = nil,
+ last_id = nil,
+ kvs = [],
+ kvs_size = 0,
+ changes = 0,
+ total_changes}).
compact(_Db, State, Opts) ->
@@ -37,9 +36,11 @@ compact(State) ->
#mrst{
db_name=DbName,
idx_name=IdxName,
+ seq_indexed=SeqIndexed,
sig=Sig,
update_seq=Seq,
id_btree=IdBtree,
+ seq_btree=SeqBtree,
views=Views
} = State,
@@ -51,16 +52,25 @@ compact(State) ->
#mrst{
id_btree = EmptyIdBtree,
+ seq_btree = EmptySeqBtree,
views = EmptyViews
} = EmptyState,
{ok, Count} = couch_btree:full_reduce(IdBtree),
- TotalChanges = lists:foldl(
+ TotalChanges0 = lists:foldl(
fun(View, Acc) ->
{ok, Kvs} = couch_mrview_util:get_row_count(View),
Acc + Kvs
end,
Count, Views),
+
+
+ TotalChanges = case SeqIndexed of
+ true ->
+ TotalChanges0 * 2;
+ _ ->
+ TotalChanges0
+ end,
couch_task_status:add_task([
{type, view_compaction},
{database, DbName},
@@ -74,7 +84,8 @@ compact(State) ->
BufferSize = list_to_integer(BufferSize0),
FoldFun = fun({DocId, _} = KV, Acc) ->
- #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize, last_id = LastId} = Acc,
+ #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize,
+ last_id = LastId} = Acc,
if DocId =:= LastId ->
% COUCHDB-999 regression test
?LOG_ERROR("Duplicate docid `~s` detected in view group `~s`"
@@ -88,31 +99,64 @@ compact(State) ->
true ->
{ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])),
Acc2 = update_task(Acc, 1 + length(Kvs)),
- {ok, Acc2#acc{
- btree = Bt2, kvs = [], kvs_size = 0, last_id = DocId}};
+ {ok, Acc2#acc{btree = Bt2, kvs = [], kvs_size = 0,
+ last_id = DocId}};
_ ->
- {ok, Acc#acc{
- kvs = [KV | Kvs], kvs_size = KvsSize2, last_id = DocId}}
+ {ok, Acc#acc{kvs = [KV | Kvs], kvs_size = KvsSize2,
+ last_id = DocId}}
end
end,
+ %% compact view group byte
InitAcc = #acc{total_changes = TotalChanges, btree = EmptyIdBtree},
{ok, _, FinalAcc} = couch_btree:foldl(IdBtree, FoldFun, InitAcc),
#acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
- {ok, NewIdBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
+ Uncopied1 = lists:reverse(Uncopied),
+ {ok, NewIdBtree} = couch_btree:add(Bt3, Uncopied1),
FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
- {NewViews, _} = lists:mapfoldl(fun({View, EmptyView}, Acc) ->
+ {NewViews, FinalAcc3} = lists:mapfoldl(fun({View, EmptyView}, Acc) ->
compact_view(View, EmptyView, BufferSize, Acc)
end, FinalAcc2, lists:zip(Views, EmptyViews)),
+ %% compact main seq btree
+ NewSeqBtree = compact_seq_btree(SeqBtree, EmptySeqBtree, BufferSize,
+ FinalAcc3),
+
unlink(EmptyState#mrst.fd),
{ok, EmptyState#mrst{
id_btree=NewIdBtree,
+ seq_btree=NewSeqBtree,
views=NewViews,
update_seq=Seq
}}.
+compact_seq_btree(nil, _, _, _) ->
+ nil;
+compact_seq_btree(SeqBtree, EmptySeqBtree, BufferSize, Acc0) ->
+ FoldFun = fun(KV, Acc) ->
+ #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc,
+
+ KvsSize2 = KvsSize + ?term_size(KV),
+ case KvsSize2 >= BufferSize of
+ true ->
+ ToAdd = lists:reverse([KV | Kvs]),
+ {ok, Bt2} = couch_btree:add(Bt, ToAdd),
+ Acc2 = update_task(Acc, 1 + length(Kvs)),
+ {ok, Acc2#acc{btree = Bt2, kvs = [], kvs_size = 0}};
+ _ ->
+ {ok, Acc#acc{kvs = [KV | Kvs], kvs_size = KvsSize2}}
+ end
+ end,
+
+ InitAcc = Acc0#acc{kvs=[], kvs_size=0, btree = EmptySeqBtree},
+ {ok, _, FinalAcc} = couch_btree:foldl(SeqBtree, FoldFun, InitAcc),
+ #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
+ Uncopied1 = lists:reverse(Uncopied),
+ {ok, NewSeqBtree} = couch_btree:add(Bt3, Uncopied1),
+ update_task(FinalAcc, length(Uncopied)),
+ NewSeqBtree.
+
recompact(State) ->
link(State#mrst.fd),
@@ -139,12 +183,28 @@ compact_view(View, EmptyView, BufferSize, Acc0) ->
end
end,
+ %% compact main view btree
InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyView#mrview.btree},
{ok, _, FinalAcc} = couch_btree:foldl(View#mrview.btree, Fun, InitAcc),
#acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
{ok, NewBt} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
- {EmptyView#mrview{btree=NewBt}, FinalAcc2}.
+
+ %% compact view seq btree
+ SInitAcc = FinalAcc2#acc{kvs = [], kvs_size = 0,
+ btree = EmptyView#mrview.seq_btree},
+ {NewSBt, FinalAcc3} = case View#mrview.seq_btree of
+ nil ->
+ {nil, FinalAcc2};
+ SeqBtree ->
+ {ok, _, SFinalAcc} = couch_btree:foldl(SeqBtree, Fun, SInitAcc),
+ #acc{btree = SBt3, kvs = SUncopied} = SFinalAcc,
+ {ok, NewSBt1} = couch_btree:add(SBt3, lists:reverse(SUncopied)),
+ SFinalAcc1 = update_task(SFinalAcc, length(SUncopied)),
+ {NewSBt1, SFinalAcc1}
+ end,
+
+ {EmptyView#mrview{btree=NewBt, seq_btree=NewSBt}, FinalAcc3}.
update_task(#acc{changes = Changes, total_changes = Total} = Acc, ChangesInc) ->
@@ -170,5 +230,5 @@ swap_compacted(OldState, NewState) ->
unlink(OldState#mrst.fd),
couch_ref_counter:drop(OldState#mrst.refc),
{ok, NewRefCounter} = couch_ref_counter:start([NewState#mrst.fd]),
-
+
{ok, NewState#mrst{refc=NewRefCounter}}.
View
5 apps/couch_mrview/src/couch_mrview_index.erl
@@ -40,10 +40,13 @@ get(Property, State) ->
LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
IncludeDeleted = couch_util:get_value(<<"include_deleted">>,
Opts, false),
+ SeqIndexed = couch_util:get_value(<<"seq_indexed">>,
+ Opts, false),
if IncDesign -> [include_design]; true -> [] end
++ if LocalSeq -> [local_seq]; true -> [] end
- ++ if IncludeDeleted -> [include_deleted]; true -> [] end;
+ ++ if IncludeDeleted -> [include_deleted]; true -> [] end
+ ++ if SeqIndexed -> [seq_indexed]; true -> [] end;
info ->
#mrst{
fd = Fd,
View
191 apps/couch_mrview/src/couch_mrview_updater.erl
@@ -54,33 +54,59 @@ start_update(Partial, State, NumChanges) ->
purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
#mrst{
+ seq_indexed=SeqIndexed,
id_btree=IdBtree,
+ seq_btree=SeqBtree,
views=Views
} = State,
Ids = [Id || {Id, _Revs} <- PurgedIdRevs],
{ok, Lookups, IdBtree2} = couch_btree:query_modify(IdBtree, Ids, [], Ids),
+ SeqBtree2 = case SeqIndexed of
+ true ->
+ ToPurge = couch_mrview_util:to_seqkvs(Lookups, []),
+ PurgeSeqs = [{ViewId, Seq} || {{ViewId, Seq}, _Id} <- ToPurge],
+ {ok, SeqBtree1} = couch_btree:add_remove(SeqBtree, [], PurgeSeqs),
+ SeqBtree1;
+ _ ->
+ nil
+ end,
+
MakeDictFun = fun
- ({ok, {DocId, ViewNumRowKeys}}, DictAcc) ->
- FoldFun = fun({ViewNum, RowKey}, DictAcc2) ->
- dict:append(ViewNum, {RowKey, DocId}, DictAcc2)
+ ({ok, {DocId, {Seq, ViewNumRowKeys}}}, DictAcc) ->
+ FoldFun = fun({ViewNum, RowKey}, {Acc, SAcc}) ->
+ Acc2 = dict:append(ViewNum, {RowKey, DocId}, Acc),
+ SAcc2 = dict:append(ViewNum, {RowKey, Seq}, SAcc),
+ {Acc2, SAcc2}
end,
lists:foldl(FoldFun, DictAcc, ViewNumRowKeys);
({not_found, _}, DictAcc) ->
DictAcc
end,
- KeysToRemove = lists:foldl(MakeDictFun, dict:new(), Lookups),
+ {KeysToRem, SKeysToRem} = lists:foldl(MakeDictFun, {dict:new(),
+ dict:new()}, Lookups),
+
+ RemKeysFun = fun(#mrview{id_num=Num, btree=Btree,
+ seq_btree=SBtree}=View) ->
- RemKeysFun = fun(#mrview{id_num=Num, btree=Btree}=View) ->
- case dict:find(Num, KeysToRemove) of
+ case dict:find(Num, KeysToRem) of
{ok, RemKeys} ->
{ok, Btree2} = couch_btree:add_remove(Btree, [], RemKeys),
+ {ok, SBtree2} = case SeqIndexed of
+ true ->
+ {ok, nil};
+ _ ->
+ {ok, RemSKeys} = dict:find(Num, SKeysToRem),
+ couch_btree:add_remove(SBtree, [], RemSKeys)
+ end,
+
NewPurgeSeq = case Btree2 /= Btree of
true -> PurgeSeq;
_ -> View#mrview.purge_seq
end,
- View#mrview{btree=Btree2, purge_seq=NewPurgeSeq};
+ View#mrview{btree=Btree2, seq_btree=SBtree2,
+ purge_seq=NewPurgeSeq};
error ->
View
end
@@ -89,6 +115,7 @@ purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
Views2 = lists:map(RemKeysFun, Views),
{ok, State#mrst{
id_btree=IdBtree2,
+ seq_btree=SeqBtree2,
views=Views2,
purge_seq=PurgeSeq
}}.
@@ -145,37 +172,45 @@ compute_map_results(#mrst{qserver = Qs}, Dequeued) ->
% Run all the non deleted docs through the view engine and
% then pass the results on to the writer process.
DocFun = fun
- ({nil, Seq, _}, {SeqAcc, AccDel, AccNotDel}) ->
- {erlang:max(Seq, SeqAcc), AccDel, AccNotDel};
- ({Id, Seq, deleted}, {SeqAcc, AccDel, AccNotDel}) ->
- {erlang:max(Seq, SeqAcc), [{Id, []} | AccDel], AccNotDel};
- ({_Id, Seq, Doc}, {SeqAcc, AccDel, AccNotDel}) ->
- {erlang:max(Seq, SeqAcc), AccDel, [Doc | AccNotDel]}
+ ({nil, Seq, _}, {SeqAcc, AccDel, AccNotDel, AccDocSeq}) ->
+ {erlang:max(Seq, SeqAcc), AccDel, AccNotDel, AccDocSeq};
+ ({Id, Seq, deleted}, {SeqAcc, AccDel, AccNotDel, AccDocSeq}) ->
+ {erlang:max(Seq, SeqAcc), [{Id, Seq, []} | AccDel],
+ AccNotDel, AccDocSeq};
+ ({Id, Seq, Doc}, {SeqAcc, AccDel, AccNotDel, AccDocSeq}) ->
+ {erlang:max(Seq, SeqAcc), AccDel, [Doc | AccNotDel],
+ [{Id, Seq} | AccDocSeq]}
end,
FoldFun = fun(Docs, Acc) ->
lists:foldl(DocFun, Acc, Docs)
end,
- {MaxSeq, DeletedResults, Docs} =
- lists:foldl(FoldFun, {0, [], []}, Dequeued),
+ {MaxSeq, DeletedResults, Docs, DocSeqs} =
+ lists:foldl(FoldFun, {0, [], [], []}, Dequeued),
{ok, MapResultList} = couch_query_servers:map_docs_raw(Qs, Docs),
NotDeletedResults = lists:zipwith(
- fun(#doc{id = Id}, MapResults) -> {Id, MapResults} end,
- Docs,
+ fun({Id, Seq}, MapResults) -> {Id, Seq, MapResults} end,
+ DocSeqs,
MapResultList),
AllMapResults = DeletedResults ++ NotDeletedResults,
update_task(length(AllMapResults)),
{ok, {MaxSeq, AllMapResults}}.
-write_results(Parent, State) ->
+write_results(Parent, #mrst{db_name=DbName, idx_name=IdxName}=State) ->
case couch_work_queue:dequeue(State#mrst.write_queue) of
closed ->
Parent ! {new_state, State};
{ok, Info} ->
EmptyKVs = [{V#mrview.id_num, []} || V <- State#mrst.views],
- {Seq, ViewKVs, DocIdKeys} = merge_results(Info, 0, EmptyKVs, []),
- NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys),
+ {Seq, ViewKVs, ViewSKVs, DocIdKeys} = merge_results(Info, 0,
+ EmptyKVs,
+ EmptyKVs, []),
+ NewState = write_kvs(State, Seq, ViewKVs, ViewSKVs, DocIdKeys),
send_partial(NewState#mrst.partial_resp_pid, NewState),
+
+ % notifify the view update
+ couch_db_update_notifier:notify({view_updated, {DbName, IdxName}}),
+
write_results(Parent, NewState)
end.
@@ -191,28 +226,37 @@ start_query_server(State) ->
State#mrst{qserver=QServer}.
-merge_results([], SeqAcc, ViewKVs, DocIdKeys) ->
- {SeqAcc, ViewKVs, DocIdKeys};
-merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys) ->
- Fun = fun(RawResults, {VKV, DIK}) ->
- merge_results(RawResults, VKV, DIK)
+merge_results([], SeqAcc, ViewKVs, ViewSKVs, DocIdKeys) ->
+ {SeqAcc, ViewKVs, ViewSKVs, DocIdKeys};
+merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, ViewSKVs,
+ DocIdKeys) ->
+ Fun = fun(RawResults, {VKV, VSKV, DIK}) ->
+ merge_results(RawResults, VKV, VSKV, DIK)
end,
- {ViewKVs1, DocIdKeys1} = lists:foldl(Fun, {ViewKVs, DocIdKeys}, Results),
- merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, DocIdKeys1).
+ {ViewKVs1, ViewSKVs1, DocIdKeys1} = lists:foldl(Fun, {ViewKVs,
+ ViewSKVs,
+ DocIdKeys},
+ Results),
+ merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, ViewSKVs1,
+ DocIdKeys1).
-merge_results({DocId, []}, ViewKVs, DocIdKeys) ->
- {ViewKVs, [{DocId, []} | DocIdKeys]};
-merge_results({DocId, RawResults}, ViewKVs, DocIdKeys) ->
+merge_results({DocId, Seq, []}, ViewKVs, ViewSKVs, DocIdKeys) ->
+ {ViewKVs, ViewSKVs, [{DocId, {Seq, []}} | DocIdKeys]};
+merge_results({DocId, Seq, RawResults}, ViewKVs, ViewSKVs, DocIdKeys) ->
JsonResults = couch_query_servers:raw_to_ejson(RawResults),
Results = [[list_to_tuple(Res) || Res <- FunRs] || FunRs <- JsonResults],
- {ViewKVs1, ViewIdKeys} = insert_results(DocId, Results, ViewKVs, [], []),
- {ViewKVs1, [ViewIdKeys | DocIdKeys]}.
+ {ViewKVs1, ViewSKVs1, ViewIdKeys} = insert_results(DocId, Seq, Results,
+ ViewKVs, ViewSKVs,
+ [], [], []),
+ {ViewKVs1, ViewSKVs1, [ViewIdKeys | DocIdKeys]}.
-insert_results(DocId, [], [], ViewKVs, ViewIdKeys) ->
- {lists:reverse(ViewKVs), {DocId, ViewIdKeys}};
-insert_results(DocId, [KVs | RKVs], [{Id, VKVs} | RVKVs], VKVAcc, VIdKeys) ->
+insert_results(DocId, Seq, [], [], [], ViewKVs, ViewSKVs, ViewIdKeys) ->
+ {lists:reverse(ViewKVs), lists:reverse(ViewSKVs), {DocId, {Seq,
+ ViewIdKeys}}};
+insert_results(DocId, Seq, [KVs | RKVs], [{Id, VKVs} | RVKVs],
+ [{Id, VSKVs} | RVSKVs], VKVAcc, VSKVAcc, VIdKeys) ->
CombineDupesFun = fun
({Key, Val}, {[{Key, {dups, Vals}} | Rest], IdKeys}) ->
{[{Key, {dups, [Val | Vals]}} | Rest], IdKeys};
@@ -224,54 +268,91 @@ insert_results(DocId, [KVs | RKVs], [{Id, VKVs} | RVKVs], VKVAcc, VIdKeys) ->
InitAcc = {[], VIdKeys},
{Duped, VIdKeys0} = lists:foldl(CombineDupesFun, InitAcc, lists:sort(KVs)),
FinalKVs = [{{Key, DocId}, Val} || {Key, Val} <- Duped] ++ VKVs,
- insert_results(DocId, RKVs, RVKVs, [{Id, FinalKVs} | VKVAcc], VIdKeys0).
+ FinalSKVs = [{{Key, Seq}, {DocId, Val}} || {Key, Val} <- Duped] ++ VSKVs,
+ insert_results(DocId, Seq, RKVs, RVKVs, RVSKVs, [{Id, FinalKVs} | VKVAcc],
+ [{Id, FinalSKVs} | VSKVAcc], VIdKeys0).
-write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys) ->
+write_kvs(State, UpdateSeq, ViewKVs, ViewSKVs, DocIdKeys) ->
#mrst{
+ seq_indexed=SeqIndexed,
id_btree=IdBtree,
+ seq_btree=SeqBtree,
first_build=FirstBuild
} = State,
{ok, ToRemove, IdBtree2} = update_id_btree(IdBtree, DocIdKeys, FirstBuild),
- ToRemByView = collapse_rem_keys(ToRemove, dict:new()),
+ {ToRemByView, ToRemByViewSeq} = collapse_rem_keys(ToRemove,
+ dict:new(),
+ dict:new()),
- UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, KVs}) ->
+ UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, KVs},
+ {ViewId, SKVs}) ->
ToRem = couch_util:dict_find(ViewId, ToRemByView, []),
{ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, KVs, ToRem),
+ {ok, VSBtree2} = case SeqIndexed of
+ true ->
+ SeqToRem = couch_util:dict_find(ViewId, ToRemByViewSeq, []),
+ couch_btree:add_remove(View#mrview.seq_btree, SKVs,
+ SeqToRem);
+ _ ->
+ {ok, nil}
+ end,
NewUpdateSeq = case VBtree2 =/= View#mrview.btree of
true -> UpdateSeq;
_ -> View#mrview.update_seq
end,
- View#mrview{btree=VBtree2, update_seq=NewUpdateSeq}
+ View#mrview{btree=VBtree2, seq_btree=VSBtree2,
+ update_seq=NewUpdateSeq}
+ end,
+
+ SeqBtree2 = case SeqIndexed of
+ true ->
+ SToRemove = couch_mrview_util:to_seqkvs(ToRemove, []),
+ ToRemBySeq = [{ViewId, Seq} || {{ViewId, Seq}, _Id} <- SToRemove],
+ {ok, SeqBtree1} = update_seq_btree(SeqBtree, ViewSKVs,
+ ToRemBySeq),
+ SeqBtree1;
+ _ ->
+ nil
end,
State#mrst{
- views=lists:zipwith(UpdateView, State#mrst.views, ViewKVs),
+ views=lists:zipwith3(UpdateView, State#mrst.views, ViewKVs, ViewSKVs),
update_seq=UpdateSeq,
- id_btree=IdBtree2
+ id_btree=IdBtree2,
+ seq_btree=SeqBtree2
}.
update_id_btree(Btree, DocIdKeys, true) ->
- ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- DocIdKeys, DIKeys /= []],
+ ToAdd = [{Id, {Seq, Keys}} || {Id, {Seq, Keys}} <- DocIdKeys, Keys /= []],
couch_btree:query_modify(Btree, [], ToAdd, []);
update_id_btree(Btree, DocIdKeys, _) ->
ToFind = [Id || {Id, _} <- DocIdKeys],
- ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- DocIdKeys, DIKeys /= []],
- ToRem = [Id || {Id, DIKeys} <- DocIdKeys, DIKeys == []],
+ ToAdd = [{Id, {Seq, Keys}} || {Id, {Seq, Keys}} <- DocIdKeys, Keys /= []],
+ ToRem = [Id || {Id, {_Seq, Keys}} <- DocIdKeys, Keys == []],
couch_btree:query_modify(Btree, ToFind, ToAdd, ToRem).
-
-collapse_rem_keys([], Acc) ->
- Acc;
-collapse_rem_keys([{ok, {DocId, ViewIdKeys}} | Rest], Acc) ->
- NewAcc = lists:foldl(fun({ViewId, Key}, Acc2) ->
- dict:append(ViewId, {Key, DocId}, Acc2)
- end, Acc, ViewIdKeys),
- collapse_rem_keys(Rest, NewAcc);
-collapse_rem_keys([{not_found, _} | Rest], Acc) ->
- collapse_rem_keys(Rest, Acc).
+update_seq_btree(Btree, ViewSKVs, ToRemBySeq) ->
+ ToAdd = lists:foldl(fun({ViewId, KVS}, Acc) ->
+ lists:foldl(fun({{Key, Seq}, {DocId, Val}}, Acc1) ->
+ [{{ViewId, Seq}, {DocId, Key, Val}} | Acc1]
+ end, Acc, KVS)
+ end, [], ViewSKVs),
+ couch_btree:add_remove(Btree, ToAdd, ToRemBySeq).
+
+collapse_rem_keys([], Acc, SAcc) ->
+ {Acc, SAcc};
+collapse_rem_keys([{ok, {DocId, {Seq, ViewIdKeys}}} | Rest], Acc, SAcc) ->
+ {NewAcc, NewSAcc} = lists:foldl(fun({ViewId, Key}, {Acc2, SAcc2}) ->
+ Acc3 = dict:append(ViewId, {Key, DocId}, Acc2),
+ SAcc3 = dict:append(ViewId, {Key, Seq}, SAcc2),
+ {Acc3, SAcc3}
+ end, {Acc, SAcc}, ViewIdKeys),
+ collapse_rem_keys(Rest, NewAcc, NewSAcc);
+collapse_rem_keys([{not_found, _} | Rest], Acc, SAcc) ->
+ collapse_rem_keys(Rest, Acc, SAcc).
send_partial(Pid, State) when is_pid(Pid) ->
View
101 apps/couch_mrview/src/couch_mrview_util.erl
@@ -24,6 +24,7 @@
-export([calculate_data_size/2]).
-export([validate_args/1]).
-export([maybe_load_doc/3, maybe_load_doc/4]).
+-export([to_seqkvs/2]).
-define(MOD, couch_mrview_index).
@@ -56,10 +57,14 @@ get_view(Db, DDoc, ViewName, Args0) ->
{Type, View, Args3} = extract_view(Lang, Args2, ViewName, Views),
check_range(Args3, view_cmp(View)),
Sig = view_sig(Db, State, View, Args3),
- {ok, {Type, View}, Sig, Args3}.
+ {ok, {Type, View}, State, Sig, Args3}.
ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
+ {DesignOpts} = couch_util:get_value(<<"options">>, Fields, {[]}),
+ SeqIndexed = couch_util:get_value(<<"seq_indexed">>, DesignOpts,
+ false),
+
MakeDict = fun({Name, {MRFuns}}, DictBySrcAcc) ->
case couch_util:get_value(<<"map">>, MRFuns) of
MapSrc when is_binary(MapSrc) ->
@@ -77,7 +82,7 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
RedFuns = [{Name, RedSrc} | View#mrview.reduce_funs],
{View#mrview.map_names, RedFuns}
end,
- View2 = View#mrview{map_names=MapNames, reduce_funs=RedSrcs},
+ View2 = View#mrview{map_names=MapNames,reduce_funs=RedSrcs},
dict:store({MapSrc, ViewOpts}, View2, DictBySrcAcc);
undefined ->
DictBySrcAcc
@@ -86,11 +91,12 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
{RawViews} = couch_util:get_value(<<"views">>, Fields, {[]}),
BySrc = lists:foldl(MakeDict, dict:new(), RawViews),
- NumViews = fun({_, View}, N) -> {View#mrview{id_num=N}, N+1} end,
+ NumViews = fun({_, View}, N) ->
+ {View#mrview{id_num=N, seq_indexed=SeqIndexed}, N+1}
+ end,
{Views, _} = lists:mapfoldl(NumViews, 0, lists:sort(dict:to_list(BySrc))),
Language = couch_util:get_value(<<"language">>, Fields, <<"javascript">>),
- {DesignOpts} = couch_util:get_value(<<"options">>, Fields, {[]}),
{RawViews} = couch_util:get_value(<<"views">>, Fields, {[]}),
Lib = couch_util:get_value(<<"lib">>, RawViews, {[]}),
IncludeDeleted = couch_util:get_value(<<"include_deleted">>, DesignOpts,
@@ -103,7 +109,8 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
views=Views,
language=Language,
design_opts=DesignOpts,
- include_deleted=IncludeDeleted
+ include_deleted=IncludeDeleted,
+ seq_indexed=SeqIndexed
},
SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)},
{ok, IdxState#mrst{sig=couch_util:md5(term_to_binary(SigInfo))}}.
@@ -155,11 +162,13 @@ view_sig(_Db, State, View, Args0) ->
Sig = State#mrst.sig,
UpdateSeq = View#mrview.update_seq,
PurgeSeq = View#mrview.purge_seq,
+ SeqIndexed = View#mrview.seq_indexed,
+
Args = Args0#mrargs{
preflight_fun=undefined,
extra=[]
},
- Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, Args}),
+ Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, SeqIndexed, Args}),
couch_index_util:hexsig(couch_util:md5(Bin)).
@@ -168,21 +177,23 @@ init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
seq=0,
purge_seq=couch_db:get_purge_seq(Db),
id_btree_state=nil,
- view_states=[{nil, 0, 0} || _ <- Views]
+ seq_btree_state=nil,
+ view_states=[{nil, nil, 0, 0} || _ <- Views]
},
init_state(Db, Fd, State, Header);
init_state(Db, Fd, State, Header) ->
- #mrst{language=Lang, views=Views} = State,
+ #mrst{language=Lang, seq_indexed=SeqIndexed, views=Views} = State,
#mrheader{
seq=Seq,
purge_seq=PurgeSeq,
id_btree_state=IdBtreeState,
+ seq_btree_state=SeqBtreeState,
view_states=ViewStates
} = Header,
StateUpdate = fun
- ({_, _, _}=St) -> St;
- (St) -> {St, 0, 0}
+ ({_, _, _, _}=St) -> St;
+ (St) -> {St, nil, 0, 0}
end,
ViewStates2 = lists:map(StateUpdate, ViewStates),
@@ -194,6 +205,15 @@ init_state(Db, Fd, State, Header) ->
IdBtOpts = [{reduce, IdReduce}, {compression, couch_db:compression(Db)}],
{ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts),
+ {ok, SeqBtree} = case SeqIndexed of
+ true ->
+ SeqBtOpts = [{compression, couch_db:compression(Db)}],
+ couch_btree:open(SeqBtreeState, Fd, SeqBtOpts);
+ _ ->
+ {ok, nil}
+ end,
+
+
OpenViewFun = fun(St, View) -> open_view(Db, Fd, Lang, St, View) end,
Views2 = lists:zipwith(OpenViewFun, ViewStates2, Views),
@@ -202,11 +222,12 @@ init_state(Db, Fd, State, Header) ->
update_seq=Seq,
purge_seq=PurgeSeq,
id_btree=IdBtree,
+ seq_btree=SeqBtree,
views=Views2
}.
-open_view(Db, Fd, Lang, {BTState, USeq, PSeq}, View) ->
+open_view(Db, Fd, Lang, {BTState, SBTState, USeq, PSeq}, View) ->
FunSrcs = [FunSrc || {_Name, FunSrc} <- View#mrview.reduce_funs],
ReduceFun =
fun(reduce, KVs) ->
@@ -231,7 +252,14 @@ open_view(Db, Fd, Lang, {BTState, USeq, PSeq}, View) ->
{compression, couch_db:compression(Db)}
],
{ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts),
- View#mrview{btree=Btree, update_seq=USeq, purge_seq=PSeq}.
+ {ok, SeqBtree} = case View#mrview.seq_indexed of
+ true ->
+ couch_btree:open(SBTState, Fd, ViewBtOpts);
+ _ ->
+ {ok, nil}
+ end,
+ View#mrview{btree=Btree, seq_btree=SeqBtree, update_seq=USeq,
+ purge_seq=PSeq}.
temp_view_to_ddoc({Props}) ->
@@ -485,21 +513,35 @@ make_header(State) ->
update_seq=Seq,
purge_seq=PurgeSeq,
id_btree=IdBtree,
+ seq_btree=SeqBtree,
views=Views
} = State,
- ViewStates = [
- {
+
+ ViewStates = lists:foldr(fun(V, Acc) ->
+ SBTState = case V#mrview.seq_indexed of
+ true ->
+ couch_btree:get_state(V#mrview.seq_btree);
+ _ ->
+ nil
+ end,
+ [{
couch_btree:get_state(V#mrview.btree),
+ SBTState,
V#mrview.update_seq,
V#mrview.purge_seq
- }
- ||
- V <- Views
- ],
+ } | Acc]
+ end, [], Views),
+
+ SeqBtreeState = case SeqBtree of
+ nil -> nil;
+ _ -> couch_btree:get_state(SeqBtree)
+ end,
+
#mrheader{
seq=Seq,
purge_seq=PurgeSeq,
id_btree_state=couch_btree:get_state(IdBtree),
+ seq_btree_state=SeqBtreeState,
view_states=ViewStates
}.
@@ -557,7 +599,7 @@ reset_state(State) ->
qserver=nil,
update_seq=0,
id_btree=nil,
- views=[View#mrview{btree=nil} || View <- State#mrst.views]
+ views=[View#mrview{btree=nil, seq_btree=nil} || View <- State#mrst.views]
}.
@@ -630,8 +672,12 @@ reverse_key_default(Key) -> Key.
calculate_data_size(IdBt, Views) ->
- SumFun = fun(#mrview{btree=Bt}, Acc) ->
- sum_btree_sizes(Acc, couch_btree:size(Bt))
+ SumFun = fun
+ (#mrview{btree=Bt, seq_btree=nil}, Acc) ->
+ sum_btree_sizes(Acc, couch_btree:size(Bt));
+ (#mrview{btree=Bt, seq_btree=SBt}, Acc) ->
+ Acc1 = sum_btree_sizes(Acc, couch_btree:size(Bt)),
+ sum_btree_sizes(Acc1, couch_btree:size(SBt))
end,
Size = lists:foldl(SumFun, couch_btree:size(IdBt), Views),
{ok, Size}.
@@ -709,3 +755,16 @@ index_of(Key, [_ | Rest], Idx) ->
mrverror(Mesg) ->
throw({query_parse_error, Mesg}).
+
+
+to_seqkvs([], Acc) ->
+ lists:reverse(Acc);
+to_seqkvs([{not_found, _} | Rest], Acc) ->
+ to_seqkvs(Rest, Acc);
+to_seqkvs([{_Id, {_Seq, []}} | Rest], Acc) ->
+ to_seqkvs(Rest, Acc);
+to_seqkvs([{Id, {Seq, Keys}} | Rest], Acc0) ->
+ Acc1 = lists:foldl(fun({ViewId, _}, Acc) ->
+ [{{ViewId, Seq}, Id} | Acc]
+ end, Acc0, Keys),
+ to_seqkvs(Rest, Acc1).
Please sign in to comment.
Something went wrong with that request. Please try again.